| { |
| "schema": "dynquant_checkpoint_v2", |
| "dynquant_core": "0.3.0", |
| "quantization_config": { |
| "quant_method": "dynquant", |
| "checkpoint_format": "dynquant-packed", |
| "schema_version": 1, |
| "version": "0.3.0", |
| "group_size": 128, |
| "symmetric": false, |
| "lm_head_quantized": false, |
| "modules_to_not_convert": [], |
| "modules": { |
| "model.embed_tokens": { |
| "bits": 8, |
| "out_features": 128000 |
| }, |
| "model.layers.0.conv.in_proj": { |
| "bits": 4, |
| "out_features": 6144 |
| }, |
| "model.layers.0.conv.out_proj": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.0.feed_forward.w1": { |
| "bits": 4, |
| "out_features": 7168 |
| }, |
| "model.layers.0.feed_forward.w2": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.0.feed_forward.w3": { |
| "bits": 4, |
| "out_features": 7168 |
| }, |
| "model.layers.1.conv.in_proj": { |
| "bits": 4, |
| "out_features": 6144 |
| }, |
| "model.layers.1.conv.out_proj": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.1.feed_forward.w1": { |
| "bits": 8, |
| "out_features": 7168 |
| }, |
| "model.layers.1.feed_forward.w2": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.1.feed_forward.w3": { |
| "bits": 8, |
| "out_features": 7168 |
| }, |
| "model.layers.10.feed_forward.experts.down_proj": { |
| "bits": 2, |
| "out_features": 65536 |
| }, |
| "model.layers.10.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "out_features": 114688 |
| }, |
| "model.layers.10.feed_forward.gate": { |
| "bits": 8, |
| "out_features": 32 |
| }, |
| "model.layers.10.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 512 |
| }, |
| "model.layers.10.self_attn.out_proj": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.10.self_attn.q_proj": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.10.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 512 |
| }, |
| "model.layers.11.conv.in_proj": { |
| "bits": 4, |
| "out_features": 6144 |
| }, |
| "model.layers.11.conv.out_proj": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.11.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "out_features": 65536 |
| }, |
| "model.layers.11.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "out_features": 114688 |
| }, |
| "model.layers.11.feed_forward.gate": { |
| "bits": 8, |
| "out_features": 32 |
| }, |
| "model.layers.12.conv.in_proj": { |
| "bits": 4, |
| "out_features": 6144 |
| }, |
| "model.layers.12.conv.out_proj": { |
| "bits": 4, |
| "out_features": 2048 |
| }, |
| "model.layers.12.feed_forward.experts.down_proj": { |
| "bits": 2, |
| "out_features": 65536 |
| }, |
| "model.layers.12.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "out_features": 114688 |
| }, |
| "model.layers.12.feed_forward.gate": { |
| "bits": 8, |
| "out_features": 32 |
| }, |
| "model.layers.13.conv.in_proj": { |
| "bits": 4, |
| "out_features": 6144 |
| }, |
| "model.layers.13.conv.out_proj": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.13.feed_forward.experts.down_proj": { |
| "bits": 2, |
| "out_features": 65536 |
| }, |
| "model.layers.13.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "out_features": 114688 |
| }, |
| "model.layers.13.feed_forward.gate": { |
| "bits": 8, |
| "out_features": 32 |
| }, |
| "model.layers.14.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "out_features": 65536 |
| }, |
| "model.layers.14.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "out_features": 114688 |
| }, |
| "model.layers.14.feed_forward.gate": { |
| "bits": 8, |
| "out_features": 32 |
| }, |
| "model.layers.14.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 512 |
| }, |
| "model.layers.14.self_attn.out_proj": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.14.self_attn.q_proj": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.14.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 512 |
| }, |
| "model.layers.15.conv.in_proj": { |
| "bits": 4, |
| "out_features": 6144 |
| }, |
| "model.layers.15.conv.out_proj": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.15.feed_forward.experts.down_proj": { |
| "bits": 2, |
| "out_features": 65536 |
| }, |
| "model.layers.15.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "out_features": 114688 |
| }, |
| "model.layers.15.feed_forward.gate": { |
| "bits": 8, |
| "out_features": 32 |
| }, |
| "model.layers.16.conv.in_proj": { |
| "bits": 4, |
| "out_features": 6144 |
| }, |
| "model.layers.16.conv.out_proj": { |
| "bits": 4, |
| "out_features": 2048 |
| }, |
| "model.layers.16.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "out_features": 65536 |
| }, |
| "model.layers.16.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "out_features": 114688 |
| }, |
| "model.layers.16.feed_forward.gate": { |
| "bits": 8, |
| "out_features": 32 |
| }, |
| "model.layers.17.conv.in_proj": { |
| "bits": 4, |
| "out_features": 6144 |
| }, |
| "model.layers.17.conv.out_proj": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.17.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "out_features": 65536 |
| }, |
| "model.layers.17.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "out_features": 114688 |
| }, |
| "model.layers.17.feed_forward.gate": { |
| "bits": 8, |
| "out_features": 32 |
| }, |
| "model.layers.18.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "out_features": 65536 |
| }, |
| "model.layers.18.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "out_features": 114688 |
| }, |
| "model.layers.18.feed_forward.gate": { |
| "bits": 8, |
| "out_features": 32 |
| }, |
| "model.layers.18.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 512 |
| }, |
| "model.layers.18.self_attn.out_proj": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.18.self_attn.q_proj": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.18.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 512 |
| }, |
| "model.layers.19.conv.in_proj": { |
| "bits": 4, |
| "out_features": 6144 |
| }, |
| "model.layers.19.conv.out_proj": { |
| "bits": 4, |
| "out_features": 2048 |
| }, |
| "model.layers.19.feed_forward.experts.down_proj": { |
| "bits": 4, |
| "out_features": 65536 |
| }, |
| "model.layers.19.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "out_features": 114688 |
| }, |
| "model.layers.19.feed_forward.gate": { |
| "bits": 8, |
| "out_features": 32 |
| }, |
| "model.layers.2.feed_forward.experts.down_proj": { |
| "bits": 4, |
| "out_features": 65536 |
| }, |
| "model.layers.2.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "out_features": 114688 |
| }, |
| "model.layers.2.feed_forward.gate": { |
| "bits": 8, |
| "out_features": 32 |
| }, |
| "model.layers.2.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 512 |
| }, |
| "model.layers.2.self_attn.out_proj": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.2.self_attn.q_proj": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.2.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 512 |
| }, |
| "model.layers.20.conv.in_proj": { |
| "bits": 4, |
| "out_features": 6144 |
| }, |
| "model.layers.20.conv.out_proj": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.20.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "out_features": 65536 |
| }, |
| "model.layers.20.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "out_features": 114688 |
| }, |
| "model.layers.20.feed_forward.gate": { |
| "bits": 8, |
| "out_features": 32 |
| }, |
| "model.layers.21.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "out_features": 65536 |
| }, |
| "model.layers.21.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "out_features": 114688 |
| }, |
| "model.layers.21.feed_forward.gate": { |
| "bits": 8, |
| "out_features": 32 |
| }, |
| "model.layers.21.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 512 |
| }, |
| "model.layers.21.self_attn.out_proj": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.21.self_attn.q_proj": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.21.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 512 |
| }, |
| "model.layers.22.conv.in_proj": { |
| "bits": 4, |
| "out_features": 6144 |
| }, |
| "model.layers.22.conv.out_proj": { |
| "bits": 4, |
| "out_features": 2048 |
| }, |
| "model.layers.22.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "out_features": 65536 |
| }, |
| "model.layers.22.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "out_features": 114688 |
| }, |
| "model.layers.22.feed_forward.gate": { |
| "bits": 8, |
| "out_features": 32 |
| }, |
| "model.layers.23.conv.in_proj": { |
| "bits": 4, |
| "out_features": 6144 |
| }, |
| "model.layers.23.conv.out_proj": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.23.feed_forward.experts.down_proj": { |
| "bits": 4, |
| "out_features": 65536 |
| }, |
| "model.layers.23.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "out_features": 114688 |
| }, |
| "model.layers.23.feed_forward.gate": { |
| "bits": 8, |
| "out_features": 32 |
| }, |
| "model.layers.3.conv.in_proj": { |
| "bits": 4, |
| "out_features": 6144 |
| }, |
| "model.layers.3.conv.out_proj": { |
| "bits": 4, |
| "out_features": 2048 |
| }, |
| "model.layers.3.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "out_features": 65536 |
| }, |
| "model.layers.3.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "out_features": 114688 |
| }, |
| "model.layers.3.feed_forward.gate": { |
| "bits": 8, |
| "out_features": 32 |
| }, |
| "model.layers.4.conv.in_proj": { |
| "bits": 4, |
| "out_features": 6144 |
| }, |
| "model.layers.4.conv.out_proj": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.4.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "out_features": 65536 |
| }, |
| "model.layers.4.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "out_features": 114688 |
| }, |
| "model.layers.4.feed_forward.gate": { |
| "bits": 8, |
| "out_features": 32 |
| }, |
| "model.layers.5.conv.in_proj": { |
| "bits": 4, |
| "out_features": 6144 |
| }, |
| "model.layers.5.conv.out_proj": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.5.feed_forward.experts.down_proj": { |
| "bits": 4, |
| "out_features": 65536 |
| }, |
| "model.layers.5.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "out_features": 114688 |
| }, |
| "model.layers.5.feed_forward.gate": { |
| "bits": 8, |
| "out_features": 32 |
| }, |
| "model.layers.6.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "out_features": 65536 |
| }, |
| "model.layers.6.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "out_features": 114688 |
| }, |
| "model.layers.6.feed_forward.gate": { |
| "bits": 8, |
| "out_features": 32 |
| }, |
| "model.layers.6.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 512 |
| }, |
| "model.layers.6.self_attn.out_proj": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.6.self_attn.q_proj": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.6.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 512 |
| }, |
| "model.layers.7.conv.in_proj": { |
| "bits": 4, |
| "out_features": 6144 |
| }, |
| "model.layers.7.conv.out_proj": { |
| "bits": 4, |
| "out_features": 2048 |
| }, |
| "model.layers.7.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "out_features": 65536 |
| }, |
| "model.layers.7.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "out_features": 114688 |
| }, |
| "model.layers.7.feed_forward.gate": { |
| "bits": 8, |
| "out_features": 32 |
| }, |
| "model.layers.8.conv.in_proj": { |
| "bits": 4, |
| "out_features": 6144 |
| }, |
| "model.layers.8.conv.out_proj": { |
| "bits": 4, |
| "out_features": 2048 |
| }, |
| "model.layers.8.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "out_features": 65536 |
| }, |
| "model.layers.8.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "out_features": 114688 |
| }, |
| "model.layers.8.feed_forward.gate": { |
| "bits": 8, |
| "out_features": 32 |
| }, |
| "model.layers.9.conv.in_proj": { |
| "bits": 8, |
| "out_features": 6144 |
| }, |
| "model.layers.9.conv.out_proj": { |
| "bits": 8, |
| "out_features": 2048 |
| }, |
| "model.layers.9.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "out_features": 65536 |
| }, |
| "model.layers.9.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "out_features": 114688 |
| }, |
| "model.layers.9.feed_forward.gate": { |
| "bits": 8, |
| "out_features": 32 |
| } |
| } |
| }, |
| "files": [ |
| "model-00001-of-00002.safetensors", |
| "model-00002-of-00002.safetensors", |
| "model.safetensors.index.json" |
| ], |
| "accounting": { |
| "packed_nbytes": 4397445120, |
| "dense_nbytes": 426240, |
| "total_nbytes": 4397871360, |
| "quantized_elements": 8467644416, |
| "average_bits": 4.154586 |
| }, |
| "tied": { |
| "lm_head": "model.embed_tokens" |
| }, |
| "expert_banks": [ |
| "model.layers.10.feed_forward.experts.down_proj", |
| "model.layers.10.feed_forward.experts.gate_up_proj", |
| "model.layers.11.feed_forward.experts.down_proj", |
| "model.layers.11.feed_forward.experts.gate_up_proj", |
| "model.layers.12.feed_forward.experts.down_proj", |
| "model.layers.12.feed_forward.experts.gate_up_proj", |
| "model.layers.13.feed_forward.experts.down_proj", |
| "model.layers.13.feed_forward.experts.gate_up_proj", |
| "model.layers.14.feed_forward.experts.down_proj", |
| "model.layers.14.feed_forward.experts.gate_up_proj", |
| "model.layers.15.feed_forward.experts.down_proj", |
| "model.layers.15.feed_forward.experts.gate_up_proj", |
| "model.layers.16.feed_forward.experts.down_proj", |
| "model.layers.16.feed_forward.experts.gate_up_proj", |
| "model.layers.17.feed_forward.experts.down_proj", |
| "model.layers.17.feed_forward.experts.gate_up_proj", |
| "model.layers.18.feed_forward.experts.down_proj", |
| "model.layers.18.feed_forward.experts.gate_up_proj", |
| "model.layers.19.feed_forward.experts.down_proj", |
| "model.layers.19.feed_forward.experts.gate_up_proj", |
| "model.layers.2.feed_forward.experts.down_proj", |
| "model.layers.2.feed_forward.experts.gate_up_proj", |
| "model.layers.20.feed_forward.experts.down_proj", |
| "model.layers.20.feed_forward.experts.gate_up_proj", |
| "model.layers.21.feed_forward.experts.down_proj", |
| "model.layers.21.feed_forward.experts.gate_up_proj", |
| "model.layers.22.feed_forward.experts.down_proj", |
| "model.layers.22.feed_forward.experts.gate_up_proj", |
| "model.layers.23.feed_forward.experts.down_proj", |
| "model.layers.23.feed_forward.experts.gate_up_proj", |
| "model.layers.3.feed_forward.experts.down_proj", |
| "model.layers.3.feed_forward.experts.gate_up_proj", |
| "model.layers.4.feed_forward.experts.down_proj", |
| "model.layers.4.feed_forward.experts.gate_up_proj", |
| "model.layers.5.feed_forward.experts.down_proj", |
| "model.layers.5.feed_forward.experts.gate_up_proj", |
| "model.layers.6.feed_forward.experts.down_proj", |
| "model.layers.6.feed_forward.experts.gate_up_proj", |
| "model.layers.7.feed_forward.experts.down_proj", |
| "model.layers.7.feed_forward.experts.gate_up_proj", |
| "model.layers.8.feed_forward.experts.down_proj", |
| "model.layers.8.feed_forward.experts.gate_up_proj", |
| "model.layers.9.feed_forward.experts.down_proj", |
| "model.layers.9.feed_forward.experts.gate_up_proj" |
| ], |
| "provenance": {}, |
| "layers": { |
| "model.embed_tokens": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 128000, |
| 2048 |
| ], |
| "num_rows": 128000, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336000, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.0.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 6144, |
| 2048 |
| ], |
| "num_rows": 6144, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6684672, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.572093, |
| "clip_improvement": 0.112224 |
| }, |
| "model.layers.0.conv.out_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.0.feed_forward.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 7168, |
| 2048 |
| ], |
| "num_rows": 7168, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7798784, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955532, |
| "clip_improvement": 0.111868 |
| }, |
| "model.layers.0.feed_forward.w2": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 7168, |
| "logical_shape": [ |
| 2048, |
| 7168 |
| ], |
| "num_rows": 2048, |
| "num_groups": 56, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 15138816, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.0.feed_forward.w3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 7168, |
| 2048 |
| ], |
| "num_rows": 7168, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7798784, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955314, |
| "clip_improvement": 0.111841 |
| }, |
| "model.layers.1.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 6144, |
| 2048 |
| ], |
| "num_rows": 6144, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6684672, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.871806, |
| "clip_improvement": 0.111893 |
| }, |
| "model.layers.1.conv.out_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.1.feed_forward.w1": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 7168, |
| 2048 |
| ], |
| "num_rows": 7168, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 15138816, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.1.feed_forward.w2": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 7168, |
| "logical_shape": [ |
| 2048, |
| 7168 |
| ], |
| "num_rows": 2048, |
| "num_groups": 56, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 15138816, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.1.feed_forward.w3": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 7168, |
| 2048 |
| ], |
| "num_rows": 7168, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 15138816, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.10.feed_forward.experts.down_proj": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1792, |
| "logical_shape": [ |
| 32, |
| 2048, |
| 1792 |
| ], |
| "num_rows": 65536, |
| "num_groups": 14, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 33030144, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999935, |
| "clip_improvement": 0.357901 |
| }, |
| "model.layers.10.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 3584, |
| 2048 |
| ], |
| "num_rows": 114688, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 124780544, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954655, |
| "clip_improvement": 0.111776 |
| }, |
| "model.layers.10.feed_forward.gate": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 2048 |
| ], |
| "num_rows": 32, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 67584, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.10.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 512, |
| 2048 |
| ], |
| "num_rows": 512, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.10.self_attn.out_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.10.self_attn.q_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.10.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 512, |
| 2048 |
| ], |
| "num_rows": 512, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.11.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 6144, |
| 2048 |
| ], |
| "num_rows": 6144, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6684672, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.957336, |
| "clip_improvement": 0.11199 |
| }, |
| "model.layers.11.conv.out_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.11.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1792, |
| "logical_shape": [ |
| 32, |
| 2048, |
| 1792 |
| ], |
| "num_rows": 65536, |
| "num_groups": 14, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 47710208, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999846, |
| "clip_improvement": 0.25855 |
| }, |
| "model.layers.11.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 3584, |
| 2048 |
| ], |
| "num_rows": 114688, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 124780544, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954993, |
| "clip_improvement": 0.111796 |
| }, |
| "model.layers.11.feed_forward.gate": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 2048 |
| ], |
| "num_rows": 32, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 67584, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.12.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 6144, |
| 2048 |
| ], |
| "num_rows": 6144, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6684672, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.956879, |
| "clip_improvement": 0.111636 |
| }, |
| "model.layers.12.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.957123, |
| "clip_improvement": 0.116481 |
| }, |
| "model.layers.12.feed_forward.experts.down_proj": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1792, |
| "logical_shape": [ |
| 32, |
| 2048, |
| 1792 |
| ], |
| "num_rows": 65536, |
| "num_groups": 14, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 33030144, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999972, |
| "clip_improvement": 0.356374 |
| }, |
| "model.layers.12.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 3584, |
| 2048 |
| ], |
| "num_rows": 114688, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 124780544, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95519, |
| "clip_improvement": 0.111706 |
| }, |
| "model.layers.12.feed_forward.gate": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 2048 |
| ], |
| "num_rows": 32, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 67584, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.13.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 6144, |
| 2048 |
| ], |
| "num_rows": 6144, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6684672, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.956635, |
| "clip_improvement": 0.111741 |
| }, |
| "model.layers.13.conv.out_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.13.feed_forward.experts.down_proj": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1792, |
| "logical_shape": [ |
| 32, |
| 2048, |
| 1792 |
| ], |
| "num_rows": 65536, |
| "num_groups": 14, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 33030144, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999821, |
| "clip_improvement": 0.354768 |
| }, |
| "model.layers.13.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 3584, |
| 2048 |
| ], |
| "num_rows": 114688, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 124780544, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95525, |
| "clip_improvement": 0.111712 |
| }, |
| "model.layers.13.feed_forward.gate": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 2048 |
| ], |
| "num_rows": 32, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 67584, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.14.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1792, |
| "logical_shape": [ |
| 32, |
| 2048, |
| 1792 |
| ], |
| "num_rows": 65536, |
| "num_groups": 14, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 47710208, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999823, |
| "clip_improvement": 0.252628 |
| }, |
| "model.layers.14.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 3584, |
| 2048 |
| ], |
| "num_rows": 114688, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 124780544, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954605, |
| "clip_improvement": 0.111767 |
| }, |
| "model.layers.14.feed_forward.gate": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 2048 |
| ], |
| "num_rows": 32, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 67584, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.14.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 512, |
| 2048 |
| ], |
| "num_rows": 512, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.14.self_attn.out_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.14.self_attn.q_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.14.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 512, |
| 2048 |
| ], |
| "num_rows": 512, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.15.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 6144, |
| 2048 |
| ], |
| "num_rows": 6144, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6684672, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.957041, |
| "clip_improvement": 0.112826 |
| }, |
| "model.layers.15.conv.out_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.15.feed_forward.experts.down_proj": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1792, |
| "logical_shape": [ |
| 32, |
| 2048, |
| 1792 |
| ], |
| "num_rows": 65536, |
| "num_groups": 14, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 33030144, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999948, |
| "clip_improvement": 0.358895 |
| }, |
| "model.layers.15.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 3584, |
| 2048 |
| ], |
| "num_rows": 114688, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 124780544, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955032, |
| "clip_improvement": 0.111768 |
| }, |
| "model.layers.15.feed_forward.gate": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 2048 |
| ], |
| "num_rows": 32, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 67584, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.16.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 6144, |
| 2048 |
| ], |
| "num_rows": 6144, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6684672, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.956919, |
| "clip_improvement": 0.112441 |
| }, |
| "model.layers.16.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.956299, |
| "clip_improvement": 0.116857 |
| }, |
| "model.layers.16.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1792, |
| "logical_shape": [ |
| 32, |
| 2048, |
| 1792 |
| ], |
| "num_rows": 65536, |
| "num_groups": 14, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 47710208, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999877, |
| "clip_improvement": 0.251375 |
| }, |
| "model.layers.16.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 3584, |
| 2048 |
| ], |
| "num_rows": 114688, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 124780544, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955086, |
| "clip_improvement": 0.111563 |
| }, |
| "model.layers.16.feed_forward.gate": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 2048 |
| ], |
| "num_rows": 32, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 67584, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.17.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 6144, |
| 2048 |
| ], |
| "num_rows": 6144, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6684672, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.956044, |
| "clip_improvement": 0.112179 |
| }, |
| "model.layers.17.conv.out_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.17.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1792, |
| "logical_shape": [ |
| 32, |
| 2048, |
| 1792 |
| ], |
| "num_rows": 65536, |
| "num_groups": 14, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 47710208, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999828, |
| "clip_improvement": 0.251676 |
| }, |
| "model.layers.17.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 3584, |
| 2048 |
| ], |
| "num_rows": 114688, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 124780544, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955352, |
| "clip_improvement": 0.111793 |
| }, |
| "model.layers.17.feed_forward.gate": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 2048 |
| ], |
| "num_rows": 32, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 67584, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.18.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1792, |
| "logical_shape": [ |
| 32, |
| 2048, |
| 1792 |
| ], |
| "num_rows": 65536, |
| "num_groups": 14, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 47710208, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999844, |
| "clip_improvement": 0.250915 |
| }, |
| "model.layers.18.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 3584, |
| 2048 |
| ], |
| "num_rows": 114688, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 124780544, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954922, |
| "clip_improvement": 0.111777 |
| }, |
| "model.layers.18.feed_forward.gate": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 2048 |
| ], |
| "num_rows": 32, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 67584, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.18.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 512, |
| 2048 |
| ], |
| "num_rows": 512, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.18.self_attn.out_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.18.self_attn.q_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.18.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 512, |
| 2048 |
| ], |
| "num_rows": 512, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.19.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 6144, |
| 2048 |
| ], |
| "num_rows": 6144, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6684672, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955027, |
| "clip_improvement": 0.111579 |
| }, |
| "model.layers.19.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.9552, |
| "clip_improvement": 0.111439 |
| }, |
| "model.layers.19.feed_forward.experts.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1792, |
| "logical_shape": [ |
| 32, |
| 2048, |
| 1792 |
| ], |
| "num_rows": 65536, |
| "num_groups": 14, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 62390272, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954504, |
| "clip_improvement": 0.111898 |
| }, |
| "model.layers.19.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 3584, |
| 2048 |
| ], |
| "num_rows": 114688, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 124780544, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954897, |
| "clip_improvement": 0.111735 |
| }, |
| "model.layers.19.feed_forward.gate": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 2048 |
| ], |
| "num_rows": 32, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 67584, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.2.feed_forward.experts.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1792, |
| "logical_shape": [ |
| 32, |
| 2048, |
| 1792 |
| ], |
| "num_rows": 65536, |
| "num_groups": 14, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 62390272, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954591, |
| "clip_improvement": 0.112094 |
| }, |
| "model.layers.2.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 3584, |
| 2048 |
| ], |
| "num_rows": 114688, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 124780544, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.945719, |
| "clip_improvement": 0.111894 |
| }, |
| "model.layers.2.feed_forward.gate": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 2048 |
| ], |
| "num_rows": 32, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 67584, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.2.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 512, |
| 2048 |
| ], |
| "num_rows": 512, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.2.self_attn.out_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.2.self_attn.q_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.2.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 512, |
| 2048 |
| ], |
| "num_rows": 512, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.20.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 6144, |
| 2048 |
| ], |
| "num_rows": 6144, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6684672, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95754, |
| "clip_improvement": 0.112456 |
| }, |
| "model.layers.20.conv.out_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.20.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1792, |
| "logical_shape": [ |
| 32, |
| 2048, |
| 1792 |
| ], |
| "num_rows": 65536, |
| "num_groups": 14, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 47710208, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999821, |
| "clip_improvement": 0.252432 |
| }, |
| "model.layers.20.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 3584, |
| 2048 |
| ], |
| "num_rows": 114688, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 124780544, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95474, |
| "clip_improvement": 0.111734 |
| }, |
| "model.layers.20.feed_forward.gate": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 2048 |
| ], |
| "num_rows": 32, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 67584, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.21.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1792, |
| "logical_shape": [ |
| 32, |
| 2048, |
| 1792 |
| ], |
| "num_rows": 65536, |
| "num_groups": 14, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 47710208, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999835, |
| "clip_improvement": 0.251336 |
| }, |
| "model.layers.21.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 3584, |
| 2048 |
| ], |
| "num_rows": 114688, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 124780544, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954689, |
| "clip_improvement": 0.111906 |
| }, |
| "model.layers.21.feed_forward.gate": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 2048 |
| ], |
| "num_rows": 32, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 67584, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.21.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 512, |
| 2048 |
| ], |
| "num_rows": 512, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.21.self_attn.out_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.21.self_attn.q_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.21.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 512, |
| 2048 |
| ], |
| "num_rows": 512, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.22.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 6144, |
| 2048 |
| ], |
| "num_rows": 6144, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6684672, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955546, |
| "clip_improvement": 0.112006 |
| }, |
| "model.layers.22.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.959747, |
| "clip_improvement": 0.111649 |
| }, |
| "model.layers.22.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1792, |
| "logical_shape": [ |
| 32, |
| 2048, |
| 1792 |
| ], |
| "num_rows": 65536, |
| "num_groups": 14, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 47710208, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999837, |
| "clip_improvement": 0.250662 |
| }, |
| "model.layers.22.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 3584, |
| 2048 |
| ], |
| "num_rows": 114688, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 124780544, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954348, |
| "clip_improvement": 0.111871 |
| }, |
| "model.layers.22.feed_forward.gate": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 2048 |
| ], |
| "num_rows": 32, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 67584, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.23.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 6144, |
| 2048 |
| ], |
| "num_rows": 6144, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6684672, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955475, |
| "clip_improvement": 0.111974 |
| }, |
| "model.layers.23.conv.out_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.23.feed_forward.experts.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1792, |
| "logical_shape": [ |
| 32, |
| 2048, |
| 1792 |
| ], |
| "num_rows": 65536, |
| "num_groups": 14, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 62390272, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.956104, |
| "clip_improvement": 0.112122 |
| }, |
| "model.layers.23.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 3584, |
| 2048 |
| ], |
| "num_rows": 114688, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 124780544, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954569, |
| "clip_improvement": 0.111799 |
| }, |
| "model.layers.23.feed_forward.gate": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 2048 |
| ], |
| "num_rows": 32, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 67584, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.3.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 6144, |
| 2048 |
| ], |
| "num_rows": 6144, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6684672, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.875854, |
| "clip_improvement": 0.111644 |
| }, |
| "model.layers.3.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.949188, |
| "clip_improvement": 0.11668 |
| }, |
| "model.layers.3.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1792, |
| "logical_shape": [ |
| 32, |
| 2048, |
| 1792 |
| ], |
| "num_rows": 65536, |
| "num_groups": 14, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 47710208, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.99986, |
| "clip_improvement": 0.25093 |
| }, |
| "model.layers.3.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 3584, |
| 2048 |
| ], |
| "num_rows": 114688, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 124780544, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954584, |
| "clip_improvement": 0.111966 |
| }, |
| "model.layers.3.feed_forward.gate": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 2048 |
| ], |
| "num_rows": 32, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 67584, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.4.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 6144, |
| 2048 |
| ], |
| "num_rows": 6144, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6684672, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.936066, |
| "clip_improvement": 0.111845 |
| }, |
| "model.layers.4.conv.out_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.4.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1792, |
| "logical_shape": [ |
| 32, |
| 2048, |
| 1792 |
| ], |
| "num_rows": 65536, |
| "num_groups": 14, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 47710208, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999846, |
| "clip_improvement": 0.254989 |
| }, |
| "model.layers.4.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 3584, |
| 2048 |
| ], |
| "num_rows": 114688, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 124780544, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954652, |
| "clip_improvement": 0.111997 |
| }, |
| "model.layers.4.feed_forward.gate": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 2048 |
| ], |
| "num_rows": 32, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 67584, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.5.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 6144, |
| 2048 |
| ], |
| "num_rows": 6144, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6684672, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.951579, |
| "clip_improvement": 0.111853 |
| }, |
| "model.layers.5.conv.out_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.5.feed_forward.experts.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1792, |
| "logical_shape": [ |
| 32, |
| 2048, |
| 1792 |
| ], |
| "num_rows": 65536, |
| "num_groups": 14, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 62390272, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954411, |
| "clip_improvement": 0.112166 |
| }, |
| "model.layers.5.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 3584, |
| 2048 |
| ], |
| "num_rows": 114688, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 124780544, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954376, |
| "clip_improvement": 0.11188 |
| }, |
| "model.layers.5.feed_forward.gate": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 2048 |
| ], |
| "num_rows": 32, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 67584, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.6.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1792, |
| "logical_shape": [ |
| 32, |
| 2048, |
| 1792 |
| ], |
| "num_rows": 65536, |
| "num_groups": 14, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 47710208, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999837, |
| "clip_improvement": 0.253612 |
| }, |
| "model.layers.6.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 3584, |
| 2048 |
| ], |
| "num_rows": 114688, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 124780544, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95448, |
| "clip_improvement": 0.111896 |
| }, |
| "model.layers.6.feed_forward.gate": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 2048 |
| ], |
| "num_rows": 32, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 67584, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.6.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 512, |
| 2048 |
| ], |
| "num_rows": 512, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.6.self_attn.out_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.6.self_attn.q_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.6.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 512, |
| 2048 |
| ], |
| "num_rows": 512, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.7.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 6144, |
| 2048 |
| ], |
| "num_rows": 6144, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6684672, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955068, |
| "clip_improvement": 0.112377 |
| }, |
| "model.layers.7.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.958038, |
| "clip_improvement": 0.115686 |
| }, |
| "model.layers.7.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1792, |
| "logical_shape": [ |
| 32, |
| 2048, |
| 1792 |
| ], |
| "num_rows": 65536, |
| "num_groups": 14, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 47710208, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999832, |
| "clip_improvement": 0.257324 |
| }, |
| "model.layers.7.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 3584, |
| 2048 |
| ], |
| "num_rows": 114688, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 124780544, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95477, |
| "clip_improvement": 0.1119 |
| }, |
| "model.layers.7.feed_forward.gate": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 2048 |
| ], |
| "num_rows": 32, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 67584, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.8.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 6144, |
| 2048 |
| ], |
| "num_rows": 6144, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6684672, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95636, |
| "clip_improvement": 0.112536 |
| }, |
| "model.layers.8.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.958008, |
| "clip_improvement": 0.113729 |
| }, |
| "model.layers.8.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1792, |
| "logical_shape": [ |
| 32, |
| 2048, |
| 1792 |
| ], |
| "num_rows": 65536, |
| "num_groups": 14, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 47710208, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999864, |
| "clip_improvement": 0.256132 |
| }, |
| "model.layers.8.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 3584, |
| 2048 |
| ], |
| "num_rows": 114688, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 124780544, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954432, |
| "clip_improvement": 0.111738 |
| }, |
| "model.layers.8.feed_forward.gate": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 2048 |
| ], |
| "num_rows": 32, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 67584, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.9.conv.in_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 6144, |
| 2048 |
| ], |
| "num_rows": 6144, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 12976128, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.9.conv.out_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "num_rows": 2048, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.9.feed_forward.experts.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1792, |
| "logical_shape": [ |
| 32, |
| 2048, |
| 1792 |
| ], |
| "num_rows": 65536, |
| "num_groups": 14, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 47710208, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999838, |
| "clip_improvement": 0.251522 |
| }, |
| "model.layers.9.feed_forward.experts.gate_up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 3584, |
| 2048 |
| ], |
| "num_rows": 114688, |
| "num_groups": 16, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 124780544, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95512, |
| "clip_improvement": 0.111789 |
| }, |
| "model.layers.9.feed_forward.gate": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 2048, |
| "logical_shape": [ |
| 32, |
| 2048 |
| ], |
| "num_rows": 32, |
| "num_groups": 16, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 67584, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| } |
| } |
| } |
