{ "schema": "dynquant_checkpoint_v2", "dynquant_core": "0.3.0", "quantization_config": { "quant_method": "dynquant", "checkpoint_format": "dynquant-packed", "schema_version": 1, "version": "0.3.0", "group_size": 128, "symmetric": false, "lm_head_quantized": false, "modules_to_not_convert": [], "modules": { "model.embed_tokens": { "bits": 8, "out_features": 128000 }, "model.layers.0.conv.in_proj": { "bits": 4, "out_features": 6144 }, "model.layers.0.conv.out_proj": { "bits": 8, "out_features": 2048 }, "model.layers.0.feed_forward.w1": { "bits": 4, "out_features": 7168 }, "model.layers.0.feed_forward.w2": { "bits": 8, "out_features": 2048 }, "model.layers.0.feed_forward.w3": { "bits": 4, "out_features": 7168 }, "model.layers.1.conv.in_proj": { "bits": 4, "out_features": 6144 }, "model.layers.1.conv.out_proj": { "bits": 8, "out_features": 2048 }, "model.layers.1.feed_forward.w1": { "bits": 8, "out_features": 7168 }, "model.layers.1.feed_forward.w2": { "bits": 8, "out_features": 2048 }, "model.layers.1.feed_forward.w3": { "bits": 8, "out_features": 7168 }, "model.layers.10.feed_forward.experts.down_proj": { "bits": 2, "out_features": 65536 }, "model.layers.10.feed_forward.experts.gate_up_proj": { "bits": 4, "out_features": 114688 }, "model.layers.10.feed_forward.gate": { "bits": 8, "out_features": 32 }, "model.layers.10.self_attn.k_proj": { "bits": 8, "out_features": 512 }, "model.layers.10.self_attn.out_proj": { "bits": 8, "out_features": 2048 }, "model.layers.10.self_attn.q_proj": { "bits": 8, "out_features": 2048 }, "model.layers.10.self_attn.v_proj": { "bits": 8, "out_features": 512 }, "model.layers.11.conv.in_proj": { "bits": 4, "out_features": 6144 }, "model.layers.11.conv.out_proj": { "bits": 8, "out_features": 2048 }, "model.layers.11.feed_forward.experts.down_proj": { "bits": 3, "out_features": 65536 }, "model.layers.11.feed_forward.experts.gate_up_proj": { "bits": 4, "out_features": 114688 }, "model.layers.11.feed_forward.gate": { "bits": 8, "out_features": 32 }, "model.layers.12.conv.in_proj": { "bits": 4, "out_features": 6144 }, "model.layers.12.conv.out_proj": { "bits": 4, "out_features": 2048 }, "model.layers.12.feed_forward.experts.down_proj": { "bits": 2, "out_features": 65536 }, "model.layers.12.feed_forward.experts.gate_up_proj": { "bits": 4, "out_features": 114688 }, "model.layers.12.feed_forward.gate": { "bits": 8, "out_features": 32 }, "model.layers.13.conv.in_proj": { "bits": 4, "out_features": 6144 }, "model.layers.13.conv.out_proj": { "bits": 8, "out_features": 2048 }, "model.layers.13.feed_forward.experts.down_proj": { "bits": 2, "out_features": 65536 }, "model.layers.13.feed_forward.experts.gate_up_proj": { "bits": 4, "out_features": 114688 }, "model.layers.13.feed_forward.gate": { "bits": 8, "out_features": 32 }, "model.layers.14.feed_forward.experts.down_proj": { "bits": 3, "out_features": 65536 }, "model.layers.14.feed_forward.experts.gate_up_proj": { "bits": 4, "out_features": 114688 }, "model.layers.14.feed_forward.gate": { "bits": 8, "out_features": 32 }, "model.layers.14.self_attn.k_proj": { "bits": 8, "out_features": 512 }, "model.layers.14.self_attn.out_proj": { "bits": 8, "out_features": 2048 }, "model.layers.14.self_attn.q_proj": { "bits": 8, "out_features": 2048 }, "model.layers.14.self_attn.v_proj": { "bits": 8, "out_features": 512 }, "model.layers.15.conv.in_proj": { "bits": 4, "out_features": 6144 }, "model.layers.15.conv.out_proj": { "bits": 8, "out_features": 2048 }, "model.layers.15.feed_forward.experts.down_proj": { "bits": 2, "out_features": 65536 }, "model.layers.15.feed_forward.experts.gate_up_proj": { "bits": 4, "out_features": 114688 }, "model.layers.15.feed_forward.gate": { "bits": 8, "out_features": 32 }, "model.layers.16.conv.in_proj": { "bits": 4, "out_features": 6144 }, "model.layers.16.conv.out_proj": { "bits": 4, "out_features": 2048 }, "model.layers.16.feed_forward.experts.down_proj": { "bits": 3, "out_features": 65536 }, "model.layers.16.feed_forward.experts.gate_up_proj": { "bits": 4, "out_features": 114688 }, "model.layers.16.feed_forward.gate": { "bits": 8, "out_features": 32 }, "model.layers.17.conv.in_proj": { "bits": 4, "out_features": 6144 }, "model.layers.17.conv.out_proj": { "bits": 8, "out_features": 2048 }, "model.layers.17.feed_forward.experts.down_proj": { "bits": 3, "out_features": 65536 }, "model.layers.17.feed_forward.experts.gate_up_proj": { "bits": 4, "out_features": 114688 }, "model.layers.17.feed_forward.gate": { "bits": 8, "out_features": 32 }, "model.layers.18.feed_forward.experts.down_proj": { "bits": 3, "out_features": 65536 }, "model.layers.18.feed_forward.experts.gate_up_proj": { "bits": 4, "out_features": 114688 }, "model.layers.18.feed_forward.gate": { "bits": 8, "out_features": 32 }, "model.layers.18.self_attn.k_proj": { "bits": 8, "out_features": 512 }, "model.layers.18.self_attn.out_proj": { "bits": 8, "out_features": 2048 }, "model.layers.18.self_attn.q_proj": { "bits": 8, "out_features": 2048 }, "model.layers.18.self_attn.v_proj": { "bits": 8, "out_features": 512 }, "model.layers.19.conv.in_proj": { "bits": 4, "out_features": 6144 }, "model.layers.19.conv.out_proj": { "bits": 4, "out_features": 2048 }, "model.layers.19.feed_forward.experts.down_proj": { "bits": 4, "out_features": 65536 }, "model.layers.19.feed_forward.experts.gate_up_proj": { "bits": 4, "out_features": 114688 }, "model.layers.19.feed_forward.gate": { "bits": 8, "out_features": 32 }, "model.layers.2.feed_forward.experts.down_proj": { "bits": 4, "out_features": 65536 }, "model.layers.2.feed_forward.experts.gate_up_proj": { "bits": 4, "out_features": 114688 }, "model.layers.2.feed_forward.gate": { "bits": 8, "out_features": 32 }, "model.layers.2.self_attn.k_proj": { "bits": 8, "out_features": 512 }, "model.layers.2.self_attn.out_proj": { "bits": 8, "out_features": 2048 }, "model.layers.2.self_attn.q_proj": { "bits": 8, "out_features": 2048 }, "model.layers.2.self_attn.v_proj": { "bits": 8, "out_features": 512 }, "model.layers.20.conv.in_proj": { "bits": 4, "out_features": 6144 }, "model.layers.20.conv.out_proj": { "bits": 8, "out_features": 2048 }, "model.layers.20.feed_forward.experts.down_proj": { "bits": 3, "out_features": 65536 }, "model.layers.20.feed_forward.experts.gate_up_proj": { "bits": 4, "out_features": 114688 }, "model.layers.20.feed_forward.gate": { "bits": 8, "out_features": 32 }, "model.layers.21.feed_forward.experts.down_proj": { "bits": 3, "out_features": 65536 }, "model.layers.21.feed_forward.experts.gate_up_proj": { "bits": 4, "out_features": 114688 }, "model.layers.21.feed_forward.gate": { "bits": 8, "out_features": 32 }, "model.layers.21.self_attn.k_proj": { "bits": 8, "out_features": 512 }, "model.layers.21.self_attn.out_proj": { "bits": 8, "out_features": 2048 }, "model.layers.21.self_attn.q_proj": { "bits": 8, "out_features": 2048 }, "model.layers.21.self_attn.v_proj": { "bits": 8, "out_features": 512 }, "model.layers.22.conv.in_proj": { "bits": 4, "out_features": 6144 }, "model.layers.22.conv.out_proj": { "bits": 4, "out_features": 2048 }, "model.layers.22.feed_forward.experts.down_proj": { "bits": 3, "out_features": 65536 }, "model.layers.22.feed_forward.experts.gate_up_proj": { "bits": 4, "out_features": 114688 }, "model.layers.22.feed_forward.gate": { "bits": 8, "out_features": 32 }, "model.layers.23.conv.in_proj": { "bits": 4, "out_features": 6144 }, "model.layers.23.conv.out_proj": { "bits": 8, "out_features": 2048 }, "model.layers.23.feed_forward.experts.down_proj": { "bits": 4, "out_features": 65536 }, "model.layers.23.feed_forward.experts.gate_up_proj": { "bits": 4, "out_features": 114688 }, "model.layers.23.feed_forward.gate": { "bits": 8, "out_features": 32 }, "model.layers.3.conv.in_proj": { "bits": 4, "out_features": 6144 }, "model.layers.3.conv.out_proj": { "bits": 4, "out_features": 2048 }, "model.layers.3.feed_forward.experts.down_proj": { "bits": 3, "out_features": 65536 }, "model.layers.3.feed_forward.experts.gate_up_proj": { "bits": 4, "out_features": 114688 }, "model.layers.3.feed_forward.gate": { "bits": 8, "out_features": 32 }, "model.layers.4.conv.in_proj": { "bits": 4, "out_features": 6144 }, "model.layers.4.conv.out_proj": { "bits": 8, "out_features": 2048 }, "model.layers.4.feed_forward.experts.down_proj": { "bits": 3, "out_features": 65536 }, "model.layers.4.feed_forward.experts.gate_up_proj": { "bits": 4, "out_features": 114688 }, "model.layers.4.feed_forward.gate": { "bits": 8, "out_features": 32 }, "model.layers.5.conv.in_proj": { "bits": 4, "out_features": 6144 }, "model.layers.5.conv.out_proj": { "bits": 8, "out_features": 2048 }, "model.layers.5.feed_forward.experts.down_proj": { "bits": 4, "out_features": 65536 }, "model.layers.5.feed_forward.experts.gate_up_proj": { "bits": 4, "out_features": 114688 }, "model.layers.5.feed_forward.gate": { "bits": 8, "out_features": 32 }, "model.layers.6.feed_forward.experts.down_proj": { "bits": 3, "out_features": 65536 }, "model.layers.6.feed_forward.experts.gate_up_proj": { "bits": 4, "out_features": 114688 }, "model.layers.6.feed_forward.gate": { "bits": 8, "out_features": 32 }, "model.layers.6.self_attn.k_proj": { "bits": 8, "out_features": 512 }, "model.layers.6.self_attn.out_proj": { "bits": 8, "out_features": 2048 }, "model.layers.6.self_attn.q_proj": { "bits": 8, "out_features": 2048 }, "model.layers.6.self_attn.v_proj": { "bits": 8, "out_features": 512 }, "model.layers.7.conv.in_proj": { "bits": 4, "out_features": 6144 }, "model.layers.7.conv.out_proj": { "bits": 4, "out_features": 2048 }, "model.layers.7.feed_forward.experts.down_proj": { "bits": 3, "out_features": 65536 }, "model.layers.7.feed_forward.experts.gate_up_proj": { "bits": 4, "out_features": 114688 }, "model.layers.7.feed_forward.gate": { "bits": 8, "out_features": 32 }, "model.layers.8.conv.in_proj": { "bits": 4, "out_features": 6144 }, "model.layers.8.conv.out_proj": { "bits": 4, "out_features": 2048 }, "model.layers.8.feed_forward.experts.down_proj": { "bits": 3, "out_features": 65536 }, "model.layers.8.feed_forward.experts.gate_up_proj": { "bits": 4, "out_features": 114688 }, "model.layers.8.feed_forward.gate": { "bits": 8, "out_features": 32 }, "model.layers.9.conv.in_proj": { "bits": 8, "out_features": 6144 }, "model.layers.9.conv.out_proj": { "bits": 8, "out_features": 2048 }, "model.layers.9.feed_forward.experts.down_proj": { "bits": 3, "out_features": 65536 }, "model.layers.9.feed_forward.experts.gate_up_proj": { "bits": 4, "out_features": 114688 }, "model.layers.9.feed_forward.gate": { "bits": 8, "out_features": 32 } } }, "files": [ "model-00001-of-00002.safetensors", "model-00002-of-00002.safetensors", "model.safetensors.index.json" ], "accounting": { "packed_nbytes": 4397445120, "dense_nbytes": 426240, "total_nbytes": 4397871360, "quantized_elements": 8467644416, "average_bits": 4.154586 }, "tied": { "lm_head": "model.embed_tokens" }, "expert_banks": [ "model.layers.10.feed_forward.experts.down_proj", "model.layers.10.feed_forward.experts.gate_up_proj", "model.layers.11.feed_forward.experts.down_proj", "model.layers.11.feed_forward.experts.gate_up_proj", "model.layers.12.feed_forward.experts.down_proj", "model.layers.12.feed_forward.experts.gate_up_proj", "model.layers.13.feed_forward.experts.down_proj", "model.layers.13.feed_forward.experts.gate_up_proj", "model.layers.14.feed_forward.experts.down_proj", "model.layers.14.feed_forward.experts.gate_up_proj", "model.layers.15.feed_forward.experts.down_proj", "model.layers.15.feed_forward.experts.gate_up_proj", "model.layers.16.feed_forward.experts.down_proj", "model.layers.16.feed_forward.experts.gate_up_proj", "model.layers.17.feed_forward.experts.down_proj", "model.layers.17.feed_forward.experts.gate_up_proj", "model.layers.18.feed_forward.experts.down_proj", "model.layers.18.feed_forward.experts.gate_up_proj", "model.layers.19.feed_forward.experts.down_proj", "model.layers.19.feed_forward.experts.gate_up_proj", "model.layers.2.feed_forward.experts.down_proj", "model.layers.2.feed_forward.experts.gate_up_proj", "model.layers.20.feed_forward.experts.down_proj", "model.layers.20.feed_forward.experts.gate_up_proj", "model.layers.21.feed_forward.experts.down_proj", "model.layers.21.feed_forward.experts.gate_up_proj", "model.layers.22.feed_forward.experts.down_proj", "model.layers.22.feed_forward.experts.gate_up_proj", "model.layers.23.feed_forward.experts.down_proj", "model.layers.23.feed_forward.experts.gate_up_proj", "model.layers.3.feed_forward.experts.down_proj", "model.layers.3.feed_forward.experts.gate_up_proj", "model.layers.4.feed_forward.experts.down_proj", "model.layers.4.feed_forward.experts.gate_up_proj", "model.layers.5.feed_forward.experts.down_proj", "model.layers.5.feed_forward.experts.gate_up_proj", "model.layers.6.feed_forward.experts.down_proj", "model.layers.6.feed_forward.experts.gate_up_proj", "model.layers.7.feed_forward.experts.down_proj", "model.layers.7.feed_forward.experts.gate_up_proj", "model.layers.8.feed_forward.experts.down_proj", "model.layers.8.feed_forward.experts.gate_up_proj", "model.layers.9.feed_forward.experts.down_proj", "model.layers.9.feed_forward.experts.gate_up_proj" ], "provenance": {}, "layers": { "model.embed_tokens": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 128000, 2048 ], "num_rows": 128000, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 270336000, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.0.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 6144, 2048 ], "num_rows": 6144, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6684672, "bits_per_weight": 4.25, "clipped_fraction": 0.572093, "clip_improvement": 0.112224 }, "model.layers.0.conv.out_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.0.feed_forward.w1": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 7168, 2048 ], "num_rows": 7168, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7798784, "bits_per_weight": 4.25, "clipped_fraction": 0.955532, "clip_improvement": 0.111868 }, "model.layers.0.feed_forward.w2": { "bits": 8, "group_size": 128, "in_features": 7168, "logical_shape": [ 2048, 7168 ], "num_rows": 2048, "num_groups": 56, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 15138816, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.0.feed_forward.w3": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 7168, 2048 ], "num_rows": 7168, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7798784, "bits_per_weight": 4.25, "clipped_fraction": 0.955314, "clip_improvement": 0.111841 }, "model.layers.1.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 6144, 2048 ], "num_rows": 6144, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6684672, "bits_per_weight": 4.25, "clipped_fraction": 0.871806, "clip_improvement": 0.111893 }, "model.layers.1.conv.out_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.1.feed_forward.w1": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 7168, 2048 ], "num_rows": 7168, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 15138816, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.1.feed_forward.w2": { "bits": 8, "group_size": 128, "in_features": 7168, "logical_shape": [ 2048, 7168 ], "num_rows": 2048, "num_groups": 56, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 15138816, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.1.feed_forward.w3": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 7168, 2048 ], "num_rows": 7168, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 15138816, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.10.feed_forward.experts.down_proj": { "bits": 2, "group_size": 128, "in_features": 1792, "logical_shape": [ 32, 2048, 1792 ], "num_rows": 65536, "num_groups": 14, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 33030144, "bits_per_weight": 2.25, "clipped_fraction": 0.999935, "clip_improvement": 0.357901 }, "model.layers.10.feed_forward.experts.gate_up_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 3584, 2048 ], "num_rows": 114688, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 124780544, "bits_per_weight": 4.25, "clipped_fraction": 0.954655, "clip_improvement": 0.111776 }, "model.layers.10.feed_forward.gate": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 2048 ], "num_rows": 32, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 67584, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.10.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 512, 2048 ], "num_rows": 512, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1081344, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.10.self_attn.out_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.10.self_attn.q_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.10.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 512, 2048 ], "num_rows": 512, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1081344, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.11.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 6144, 2048 ], "num_rows": 6144, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6684672, "bits_per_weight": 4.25, "clipped_fraction": 0.957336, "clip_improvement": 0.11199 }, "model.layers.11.conv.out_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.11.feed_forward.experts.down_proj": { "bits": 3, "group_size": 128, "in_features": 1792, "logical_shape": [ 32, 2048, 1792 ], "num_rows": 65536, "num_groups": 14, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47710208, "bits_per_weight": 3.25, "clipped_fraction": 0.999846, "clip_improvement": 0.25855 }, "model.layers.11.feed_forward.experts.gate_up_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 3584, 2048 ], "num_rows": 114688, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 124780544, "bits_per_weight": 4.25, "clipped_fraction": 0.954993, "clip_improvement": 0.111796 }, "model.layers.11.feed_forward.gate": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 2048 ], "num_rows": 32, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 67584, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.12.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 6144, 2048 ], "num_rows": 6144, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6684672, "bits_per_weight": 4.25, "clipped_fraction": 0.956879, "clip_improvement": 0.111636 }, "model.layers.12.conv.out_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.957123, "clip_improvement": 0.116481 }, "model.layers.12.feed_forward.experts.down_proj": { "bits": 2, "group_size": 128, "in_features": 1792, "logical_shape": [ 32, 2048, 1792 ], "num_rows": 65536, "num_groups": 14, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 33030144, "bits_per_weight": 2.25, "clipped_fraction": 0.999972, "clip_improvement": 0.356374 }, "model.layers.12.feed_forward.experts.gate_up_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 3584, 2048 ], "num_rows": 114688, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 124780544, "bits_per_weight": 4.25, "clipped_fraction": 0.95519, "clip_improvement": 0.111706 }, "model.layers.12.feed_forward.gate": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 2048 ], "num_rows": 32, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 67584, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.13.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 6144, 2048 ], "num_rows": 6144, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6684672, "bits_per_weight": 4.25, "clipped_fraction": 0.956635, "clip_improvement": 0.111741 }, "model.layers.13.conv.out_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.13.feed_forward.experts.down_proj": { "bits": 2, "group_size": 128, "in_features": 1792, "logical_shape": [ 32, 2048, 1792 ], "num_rows": 65536, "num_groups": 14, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 33030144, "bits_per_weight": 2.25, "clipped_fraction": 0.999821, "clip_improvement": 0.354768 }, "model.layers.13.feed_forward.experts.gate_up_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 3584, 2048 ], "num_rows": 114688, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 124780544, "bits_per_weight": 4.25, "clipped_fraction": 0.95525, "clip_improvement": 0.111712 }, "model.layers.13.feed_forward.gate": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 2048 ], "num_rows": 32, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 67584, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.14.feed_forward.experts.down_proj": { "bits": 3, "group_size": 128, "in_features": 1792, "logical_shape": [ 32, 2048, 1792 ], "num_rows": 65536, "num_groups": 14, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47710208, "bits_per_weight": 3.25, "clipped_fraction": 0.999823, "clip_improvement": 0.252628 }, "model.layers.14.feed_forward.experts.gate_up_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 3584, 2048 ], "num_rows": 114688, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 124780544, "bits_per_weight": 4.25, "clipped_fraction": 0.954605, "clip_improvement": 0.111767 }, "model.layers.14.feed_forward.gate": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 2048 ], "num_rows": 32, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 67584, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.14.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 512, 2048 ], "num_rows": 512, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1081344, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.14.self_attn.out_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.14.self_attn.q_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.14.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 512, 2048 ], "num_rows": 512, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1081344, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.15.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 6144, 2048 ], "num_rows": 6144, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6684672, "bits_per_weight": 4.25, "clipped_fraction": 0.957041, "clip_improvement": 0.112826 }, "model.layers.15.conv.out_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.15.feed_forward.experts.down_proj": { "bits": 2, "group_size": 128, "in_features": 1792, "logical_shape": [ 32, 2048, 1792 ], "num_rows": 65536, "num_groups": 14, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 33030144, "bits_per_weight": 2.25, "clipped_fraction": 0.999948, "clip_improvement": 0.358895 }, "model.layers.15.feed_forward.experts.gate_up_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 3584, 2048 ], "num_rows": 114688, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 124780544, "bits_per_weight": 4.25, "clipped_fraction": 0.955032, "clip_improvement": 0.111768 }, "model.layers.15.feed_forward.gate": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 2048 ], "num_rows": 32, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 67584, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.16.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 6144, 2048 ], "num_rows": 6144, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6684672, "bits_per_weight": 4.25, "clipped_fraction": 0.956919, "clip_improvement": 0.112441 }, "model.layers.16.conv.out_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.956299, "clip_improvement": 0.116857 }, "model.layers.16.feed_forward.experts.down_proj": { "bits": 3, "group_size": 128, "in_features": 1792, "logical_shape": [ 32, 2048, 1792 ], "num_rows": 65536, "num_groups": 14, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47710208, "bits_per_weight": 3.25, "clipped_fraction": 0.999877, "clip_improvement": 0.251375 }, "model.layers.16.feed_forward.experts.gate_up_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 3584, 2048 ], "num_rows": 114688, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 124780544, "bits_per_weight": 4.25, "clipped_fraction": 0.955086, "clip_improvement": 0.111563 }, "model.layers.16.feed_forward.gate": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 2048 ], "num_rows": 32, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 67584, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.17.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 6144, 2048 ], "num_rows": 6144, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6684672, "bits_per_weight": 4.25, "clipped_fraction": 0.956044, "clip_improvement": 0.112179 }, "model.layers.17.conv.out_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.17.feed_forward.experts.down_proj": { "bits": 3, "group_size": 128, "in_features": 1792, "logical_shape": [ 32, 2048, 1792 ], "num_rows": 65536, "num_groups": 14, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47710208, "bits_per_weight": 3.25, "clipped_fraction": 0.999828, "clip_improvement": 0.251676 }, "model.layers.17.feed_forward.experts.gate_up_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 3584, 2048 ], "num_rows": 114688, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 124780544, "bits_per_weight": 4.25, "clipped_fraction": 0.955352, "clip_improvement": 0.111793 }, "model.layers.17.feed_forward.gate": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 2048 ], "num_rows": 32, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 67584, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.18.feed_forward.experts.down_proj": { "bits": 3, "group_size": 128, "in_features": 1792, "logical_shape": [ 32, 2048, 1792 ], "num_rows": 65536, "num_groups": 14, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47710208, "bits_per_weight": 3.25, "clipped_fraction": 0.999844, "clip_improvement": 0.250915 }, "model.layers.18.feed_forward.experts.gate_up_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 3584, 2048 ], "num_rows": 114688, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 124780544, "bits_per_weight": 4.25, "clipped_fraction": 0.954922, "clip_improvement": 0.111777 }, "model.layers.18.feed_forward.gate": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 2048 ], "num_rows": 32, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 67584, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.18.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 512, 2048 ], "num_rows": 512, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1081344, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.18.self_attn.out_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.18.self_attn.q_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.18.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 512, 2048 ], "num_rows": 512, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1081344, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.19.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 6144, 2048 ], "num_rows": 6144, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6684672, "bits_per_weight": 4.25, "clipped_fraction": 0.955027, "clip_improvement": 0.111579 }, "model.layers.19.conv.out_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.9552, "clip_improvement": 0.111439 }, "model.layers.19.feed_forward.experts.down_proj": { "bits": 4, "group_size": 128, "in_features": 1792, "logical_shape": [ 32, 2048, 1792 ], "num_rows": 65536, "num_groups": 14, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 62390272, "bits_per_weight": 4.25, "clipped_fraction": 0.954504, "clip_improvement": 0.111898 }, "model.layers.19.feed_forward.experts.gate_up_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 3584, 2048 ], "num_rows": 114688, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 124780544, "bits_per_weight": 4.25, "clipped_fraction": 0.954897, "clip_improvement": 0.111735 }, "model.layers.19.feed_forward.gate": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 2048 ], "num_rows": 32, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 67584, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.2.feed_forward.experts.down_proj": { "bits": 4, "group_size": 128, "in_features": 1792, "logical_shape": [ 32, 2048, 1792 ], "num_rows": 65536, "num_groups": 14, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 62390272, "bits_per_weight": 4.25, "clipped_fraction": 0.954591, "clip_improvement": 0.112094 }, "model.layers.2.feed_forward.experts.gate_up_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 3584, 2048 ], "num_rows": 114688, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 124780544, "bits_per_weight": 4.25, "clipped_fraction": 0.945719, "clip_improvement": 0.111894 }, "model.layers.2.feed_forward.gate": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 2048 ], "num_rows": 32, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 67584, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.2.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 512, 2048 ], "num_rows": 512, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1081344, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.2.self_attn.out_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.2.self_attn.q_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.2.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 512, 2048 ], "num_rows": 512, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1081344, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.20.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 6144, 2048 ], "num_rows": 6144, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6684672, "bits_per_weight": 4.25, "clipped_fraction": 0.95754, "clip_improvement": 0.112456 }, "model.layers.20.conv.out_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.20.feed_forward.experts.down_proj": { "bits": 3, "group_size": 128, "in_features": 1792, "logical_shape": [ 32, 2048, 1792 ], "num_rows": 65536, "num_groups": 14, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47710208, "bits_per_weight": 3.25, "clipped_fraction": 0.999821, "clip_improvement": 0.252432 }, "model.layers.20.feed_forward.experts.gate_up_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 3584, 2048 ], "num_rows": 114688, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 124780544, "bits_per_weight": 4.25, "clipped_fraction": 0.95474, "clip_improvement": 0.111734 }, "model.layers.20.feed_forward.gate": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 2048 ], "num_rows": 32, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 67584, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.21.feed_forward.experts.down_proj": { "bits": 3, "group_size": 128, "in_features": 1792, "logical_shape": [ 32, 2048, 1792 ], "num_rows": 65536, "num_groups": 14, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47710208, "bits_per_weight": 3.25, "clipped_fraction": 0.999835, "clip_improvement": 0.251336 }, "model.layers.21.feed_forward.experts.gate_up_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 3584, 2048 ], "num_rows": 114688, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 124780544, "bits_per_weight": 4.25, "clipped_fraction": 0.954689, "clip_improvement": 0.111906 }, "model.layers.21.feed_forward.gate": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 2048 ], "num_rows": 32, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 67584, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.21.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 512, 2048 ], "num_rows": 512, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1081344, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.21.self_attn.out_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.21.self_attn.q_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.21.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 512, 2048 ], "num_rows": 512, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1081344, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.22.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 6144, 2048 ], "num_rows": 6144, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6684672, "bits_per_weight": 4.25, "clipped_fraction": 0.955546, "clip_improvement": 0.112006 }, "model.layers.22.conv.out_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.959747, "clip_improvement": 0.111649 }, "model.layers.22.feed_forward.experts.down_proj": { "bits": 3, "group_size": 128, "in_features": 1792, "logical_shape": [ 32, 2048, 1792 ], "num_rows": 65536, "num_groups": 14, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47710208, "bits_per_weight": 3.25, "clipped_fraction": 0.999837, "clip_improvement": 0.250662 }, "model.layers.22.feed_forward.experts.gate_up_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 3584, 2048 ], "num_rows": 114688, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 124780544, "bits_per_weight": 4.25, "clipped_fraction": 0.954348, "clip_improvement": 0.111871 }, "model.layers.22.feed_forward.gate": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 2048 ], "num_rows": 32, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 67584, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.23.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 6144, 2048 ], "num_rows": 6144, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6684672, "bits_per_weight": 4.25, "clipped_fraction": 0.955475, "clip_improvement": 0.111974 }, "model.layers.23.conv.out_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.23.feed_forward.experts.down_proj": { "bits": 4, "group_size": 128, "in_features": 1792, "logical_shape": [ 32, 2048, 1792 ], "num_rows": 65536, "num_groups": 14, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 62390272, "bits_per_weight": 4.25, "clipped_fraction": 0.956104, "clip_improvement": 0.112122 }, "model.layers.23.feed_forward.experts.gate_up_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 3584, 2048 ], "num_rows": 114688, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 124780544, "bits_per_weight": 4.25, "clipped_fraction": 0.954569, "clip_improvement": 0.111799 }, "model.layers.23.feed_forward.gate": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 2048 ], "num_rows": 32, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 67584, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.3.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 6144, 2048 ], "num_rows": 6144, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6684672, "bits_per_weight": 4.25, "clipped_fraction": 0.875854, "clip_improvement": 0.111644 }, "model.layers.3.conv.out_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.949188, "clip_improvement": 0.11668 }, "model.layers.3.feed_forward.experts.down_proj": { "bits": 3, "group_size": 128, "in_features": 1792, "logical_shape": [ 32, 2048, 1792 ], "num_rows": 65536, "num_groups": 14, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47710208, "bits_per_weight": 3.25, "clipped_fraction": 0.99986, "clip_improvement": 0.25093 }, "model.layers.3.feed_forward.experts.gate_up_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 3584, 2048 ], "num_rows": 114688, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 124780544, "bits_per_weight": 4.25, "clipped_fraction": 0.954584, "clip_improvement": 0.111966 }, "model.layers.3.feed_forward.gate": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 2048 ], "num_rows": 32, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 67584, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.4.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 6144, 2048 ], "num_rows": 6144, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6684672, "bits_per_weight": 4.25, "clipped_fraction": 0.936066, "clip_improvement": 0.111845 }, "model.layers.4.conv.out_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.4.feed_forward.experts.down_proj": { "bits": 3, "group_size": 128, "in_features": 1792, "logical_shape": [ 32, 2048, 1792 ], "num_rows": 65536, "num_groups": 14, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47710208, "bits_per_weight": 3.25, "clipped_fraction": 0.999846, "clip_improvement": 0.254989 }, "model.layers.4.feed_forward.experts.gate_up_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 3584, 2048 ], "num_rows": 114688, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 124780544, "bits_per_weight": 4.25, "clipped_fraction": 0.954652, "clip_improvement": 0.111997 }, "model.layers.4.feed_forward.gate": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 2048 ], "num_rows": 32, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 67584, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.5.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 6144, 2048 ], "num_rows": 6144, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6684672, "bits_per_weight": 4.25, "clipped_fraction": 0.951579, "clip_improvement": 0.111853 }, "model.layers.5.conv.out_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.5.feed_forward.experts.down_proj": { "bits": 4, "group_size": 128, "in_features": 1792, "logical_shape": [ 32, 2048, 1792 ], "num_rows": 65536, "num_groups": 14, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 62390272, "bits_per_weight": 4.25, "clipped_fraction": 0.954411, "clip_improvement": 0.112166 }, "model.layers.5.feed_forward.experts.gate_up_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 3584, 2048 ], "num_rows": 114688, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 124780544, "bits_per_weight": 4.25, "clipped_fraction": 0.954376, "clip_improvement": 0.11188 }, "model.layers.5.feed_forward.gate": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 2048 ], "num_rows": 32, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 67584, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.6.feed_forward.experts.down_proj": { "bits": 3, "group_size": 128, "in_features": 1792, "logical_shape": [ 32, 2048, 1792 ], "num_rows": 65536, "num_groups": 14, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47710208, "bits_per_weight": 3.25, "clipped_fraction": 0.999837, "clip_improvement": 0.253612 }, "model.layers.6.feed_forward.experts.gate_up_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 3584, 2048 ], "num_rows": 114688, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 124780544, "bits_per_weight": 4.25, "clipped_fraction": 0.95448, "clip_improvement": 0.111896 }, "model.layers.6.feed_forward.gate": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 2048 ], "num_rows": 32, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 67584, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.6.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 512, 2048 ], "num_rows": 512, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1081344, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.6.self_attn.out_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.6.self_attn.q_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.6.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 512, 2048 ], "num_rows": 512, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1081344, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.7.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 6144, 2048 ], "num_rows": 6144, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6684672, "bits_per_weight": 4.25, "clipped_fraction": 0.955068, "clip_improvement": 0.112377 }, "model.layers.7.conv.out_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.958038, "clip_improvement": 0.115686 }, "model.layers.7.feed_forward.experts.down_proj": { "bits": 3, "group_size": 128, "in_features": 1792, "logical_shape": [ 32, 2048, 1792 ], "num_rows": 65536, "num_groups": 14, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47710208, "bits_per_weight": 3.25, "clipped_fraction": 0.999832, "clip_improvement": 0.257324 }, "model.layers.7.feed_forward.experts.gate_up_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 3584, 2048 ], "num_rows": 114688, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 124780544, "bits_per_weight": 4.25, "clipped_fraction": 0.95477, "clip_improvement": 0.1119 }, "model.layers.7.feed_forward.gate": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 2048 ], "num_rows": 32, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 67584, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.8.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 6144, 2048 ], "num_rows": 6144, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6684672, "bits_per_weight": 4.25, "clipped_fraction": 0.95636, "clip_improvement": 0.112536 }, "model.layers.8.conv.out_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.958008, "clip_improvement": 0.113729 }, "model.layers.8.feed_forward.experts.down_proj": { "bits": 3, "group_size": 128, "in_features": 1792, "logical_shape": [ 32, 2048, 1792 ], "num_rows": 65536, "num_groups": 14, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47710208, "bits_per_weight": 3.25, "clipped_fraction": 0.999864, "clip_improvement": 0.256132 }, "model.layers.8.feed_forward.experts.gate_up_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 3584, 2048 ], "num_rows": 114688, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 124780544, "bits_per_weight": 4.25, "clipped_fraction": 0.954432, "clip_improvement": 0.111738 }, "model.layers.8.feed_forward.gate": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 2048 ], "num_rows": 32, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 67584, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.9.conv.in_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 6144, 2048 ], "num_rows": 6144, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12976128, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.9.conv.out_proj": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 2048, 2048 ], "num_rows": 2048, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.9.feed_forward.experts.down_proj": { "bits": 3, "group_size": 128, "in_features": 1792, "logical_shape": [ 32, 2048, 1792 ], "num_rows": 65536, "num_groups": 14, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47710208, "bits_per_weight": 3.25, "clipped_fraction": 0.999838, "clip_improvement": 0.251522 }, "model.layers.9.feed_forward.experts.gate_up_proj": { "bits": 4, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 3584, 2048 ], "num_rows": 114688, "num_groups": 16, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 124780544, "bits_per_weight": 4.25, "clipped_fraction": 0.95512, "clip_improvement": 0.111789 }, "model.layers.9.feed_forward.gate": { "bits": 8, "group_size": 128, "in_features": 2048, "logical_shape": [ 32, 2048 ], "num_rows": 32, "num_groups": 16, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 67584, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 } } }