{ "schema": "dynquant_checkpoint_v2", "dynquant_core": "0.4.0", "quantization_config": { "quant_method": "dynquant", "checkpoint_format": "dynquant-packed", "schema_version": 1, "version": "0.4.0", "group_size": 128, "symmetric": false, "lm_head_quantized": true, "modules_to_not_convert": [], "modules": { "lm_head": { "bits": 8, "out_features": 32768 }, "model.embed_tokens": { "bits": 4, "out_features": 32768 }, "model.layers.0.mlp.down_proj": { "bits": 4, "out_features": 4096 }, "model.layers.0.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.0.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.0.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.0.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.0.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.0.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.1.mlp.down_proj": { "bits": 8, "out_features": 4096 }, "model.layers.1.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.1.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.1.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.1.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.1.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.1.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.10.mlp.down_proj": { "bits": 4, "out_features": 4096 }, "model.layers.10.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.10.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.10.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.10.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.10.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.10.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.11.mlp.down_proj": { "bits": 4, "out_features": 4096 }, "model.layers.11.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.11.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.11.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.11.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.11.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.11.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.12.mlp.down_proj": { "bits": 4, "out_features": 4096 }, "model.layers.12.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.12.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.12.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.12.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.12.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.12.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.13.mlp.down_proj": { "bits": 4, "out_features": 4096 }, "model.layers.13.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.13.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.13.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.13.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.13.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.13.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.14.mlp.down_proj": { "bits": 4, "out_features": 4096 }, "model.layers.14.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.14.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.14.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.14.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.14.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.14.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.15.mlp.down_proj": { "bits": 4, "out_features": 4096 }, "model.layers.15.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.15.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.15.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.15.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.15.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.15.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.16.mlp.down_proj": { "bits": 4, "out_features": 4096 }, "model.layers.16.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.16.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.16.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.16.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.16.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.16.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.17.mlp.down_proj": { "bits": 4, "out_features": 4096 }, "model.layers.17.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.17.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.17.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.17.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.17.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.17.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.18.mlp.down_proj": { "bits": 4, "out_features": 4096 }, "model.layers.18.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.18.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.18.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.18.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.18.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.18.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.19.mlp.down_proj": { "bits": 4, "out_features": 4096 }, "model.layers.19.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.19.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.19.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.19.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.19.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.19.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.2.mlp.down_proj": { "bits": 4, "out_features": 4096 }, "model.layers.2.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.2.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.2.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.2.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.2.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.2.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.20.mlp.down_proj": { "bits": 4, "out_features": 4096 }, "model.layers.20.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.20.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.20.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.20.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.20.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.20.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.21.mlp.down_proj": { "bits": 4, "out_features": 4096 }, "model.layers.21.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.21.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.21.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.21.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.21.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.21.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.22.mlp.down_proj": { "bits": 4, "out_features": 4096 }, "model.layers.22.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.22.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.22.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.22.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.22.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.22.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.23.mlp.down_proj": { "bits": 4, "out_features": 4096 }, "model.layers.23.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.23.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.23.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.23.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.23.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.23.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.24.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.24.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.24.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.24.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.24.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.24.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.24.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.25.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.25.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.25.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.25.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.25.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.25.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.25.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.26.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.26.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.26.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.26.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.26.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.26.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.26.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.27.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.27.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.27.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.27.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.27.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.27.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.27.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.28.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.28.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.28.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.28.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.28.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.28.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.28.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.29.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.29.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.29.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.29.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.29.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.29.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.29.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.3.mlp.down_proj": { "bits": 4, "out_features": 4096 }, "model.layers.3.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.3.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.3.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.3.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.3.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.3.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.30.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.30.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.30.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.30.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.30.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.30.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.30.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.31.mlp.down_proj": { "bits": 4, "out_features": 4096 }, "model.layers.31.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.31.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.31.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.31.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.31.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.31.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.4.mlp.down_proj": { "bits": 4, "out_features": 4096 }, "model.layers.4.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.4.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.4.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.4.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.4.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.4.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.5.mlp.down_proj": { "bits": 4, "out_features": 4096 }, "model.layers.5.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.5.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.5.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.5.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.5.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.5.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.6.mlp.down_proj": { "bits": 4, "out_features": 4096 }, "model.layers.6.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.6.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.6.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.6.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.6.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.6.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.7.mlp.down_proj": { "bits": 4, "out_features": 4096 }, "model.layers.7.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.7.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.7.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.7.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.7.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.7.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.8.mlp.down_proj": { "bits": 4, "out_features": 4096 }, "model.layers.8.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.8.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.8.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.8.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.8.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.8.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.9.mlp.down_proj": { "bits": 4, "out_features": 4096 }, "model.layers.9.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.9.mlp.up_proj": { "bits": 4, "out_features": 14336 }, "model.layers.9.self_attn.k_proj": { "bits": 8, "out_features": 1024 }, "model.layers.9.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.9.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.9.self_attn.v_proj": { "bits": 8, "out_features": 1024 } } }, "files": [ "model.safetensors" ], "accounting": { "packed_nbytes": 3963617280, "dense_nbytes": 532480, "total_nbytes": 3964149760, "quantized_elements": 7247757312, "average_bits": 4.375 }, "tied": {}, "expert_banks": [], "provenance": { "dynquant_core": "0.4.0", "model": "/workspace/runs/s4/mistral7b-v03.text2sql/merged", "stats": null, "allocator": "sensitivity", "widths_from": "/workspace/runs/s4/panel-mistral/maps/dq_4b.json [3964674048]", "predicted_nbytes": 3964149760 }, "layers": { "lm_head": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 32768, 4096 ], "num_rows": 32768, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 138412032, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.embed_tokens": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 32768, 4096 ], "num_rows": 32768, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 71303168, "bits_per_weight": 4.25, "clipped_fraction": 0.9277, "clip_improvement": 0.111654 }, "model.layers.0.mlp.down_proj": { "bits": 4, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954995, "clip_improvement": 0.111722 }, "model.layers.0.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.955314, "clip_improvement": 0.111629 }, "model.layers.0.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999852, "clip_improvement": 0.250389 }, "model.layers.0.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.89209, "clip_improvement": 0.21042 }, "model.layers.0.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.955269, "clip_improvement": 0.111095 }, "model.layers.0.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.870163, "clip_improvement": 0.248147 }, "model.layers.0.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.1.mlp.down_proj": { "bits": 8, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 60555264, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.1.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.95469, "clip_improvement": 0.111778 }, "model.layers.1.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.99976, "clip_improvement": 0.250698 }, "model.layers.1.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.1.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954872, "clip_improvement": 0.111292 }, "model.layers.1.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.926826, "clip_improvement": 0.134538 }, "model.layers.1.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.10.mlp.down_proj": { "bits": 4, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.955401, "clip_improvement": 0.111817 }, "model.layers.10.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.95517, "clip_improvement": 0.111777 }, "model.layers.10.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954806, "clip_improvement": 0.111762 }, "model.layers.10.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.10.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954659, "clip_improvement": 0.111894 }, "model.layers.10.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.955727, "clip_improvement": 0.111726 }, "model.layers.10.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.11.mlp.down_proj": { "bits": 4, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.955065, "clip_improvement": 0.111732 }, "model.layers.11.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.955216, "clip_improvement": 0.111866 }, "model.layers.11.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.955654, "clip_improvement": 0.111685 }, "model.layers.11.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.11.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954651, "clip_improvement": 0.111641 }, "model.layers.11.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.9543, "clip_improvement": 0.11162 }, "model.layers.11.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.12.mlp.down_proj": { "bits": 4, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954756, "clip_improvement": 0.111839 }, "model.layers.12.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954989, "clip_improvement": 0.111947 }, "model.layers.12.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.955111, "clip_improvement": 0.111493 }, "model.layers.12.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.12.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954231, "clip_improvement": 0.111711 }, "model.layers.12.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954811, "clip_improvement": 0.110958 }, "model.layers.12.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.13.mlp.down_proj": { "bits": 4, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.955226, "clip_improvement": 0.111729 }, "model.layers.13.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.955662, "clip_improvement": 0.111942 }, "model.layers.13.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.95534, "clip_improvement": 0.111718 }, "model.layers.13.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.13.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954491, "clip_improvement": 0.11161 }, "model.layers.13.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.955147, "clip_improvement": 0.111867 }, "model.layers.13.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.14.mlp.down_proj": { "bits": 4, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954919, "clip_improvement": 0.111965 }, "model.layers.14.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.955198, "clip_improvement": 0.11176 }, "model.layers.14.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.955808, "clip_improvement": 0.111607 }, "model.layers.14.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.14.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954231, "clip_improvement": 0.111575 }, "model.layers.14.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.95491, "clip_improvement": 0.111332 }, "model.layers.14.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.15.mlp.down_proj": { "bits": 4, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.95513, "clip_improvement": 0.111781 }, "model.layers.15.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.955144, "clip_improvement": 0.112064 }, "model.layers.15.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954671, "clip_improvement": 0.111558 }, "model.layers.15.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.15.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954422, "clip_improvement": 0.11157 }, "model.layers.15.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954803, "clip_improvement": 0.111575 }, "model.layers.15.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.16.mlp.down_proj": { "bits": 4, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954777, "clip_improvement": 0.112005 }, "model.layers.16.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.955168, "clip_improvement": 0.11189 }, "model.layers.16.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954926, "clip_improvement": 0.111514 }, "model.layers.16.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.16.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954048, "clip_improvement": 0.111673 }, "model.layers.16.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954651, "clip_improvement": 0.111957 }, "model.layers.16.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.17.mlp.down_proj": { "bits": 4, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.955083, "clip_improvement": 0.111797 }, "model.layers.17.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954817, "clip_improvement": 0.111652 }, "model.layers.17.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954939, "clip_improvement": 0.11166 }, "model.layers.17.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.17.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.95369, "clip_improvement": 0.111864 }, "model.layers.17.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954697, "clip_improvement": 0.111357 }, "model.layers.17.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.18.mlp.down_proj": { "bits": 4, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.95503, "clip_improvement": 0.111924 }, "model.layers.18.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954821, "clip_improvement": 0.11173 }, "model.layers.18.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.955067, "clip_improvement": 0.111609 }, "model.layers.18.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.18.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.953995, "clip_improvement": 0.111652 }, "model.layers.18.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.955261, "clip_improvement": 0.111183 }, "model.layers.18.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.19.mlp.down_proj": { "bits": 4, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954422, "clip_improvement": 0.111709 }, "model.layers.19.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954505, "clip_improvement": 0.111557 }, "model.layers.19.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954378, "clip_improvement": 0.111655 }, "model.layers.19.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.19.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.95388, "clip_improvement": 0.112022 }, "model.layers.19.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.953545, "clip_improvement": 0.111476 }, "model.layers.19.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.2.mlp.down_proj": { "bits": 4, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954793, "clip_improvement": 0.111832 }, "model.layers.2.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954213, "clip_improvement": 0.11166 }, "model.layers.2.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954668, "clip_improvement": 0.111505 }, "model.layers.2.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.2.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954819, "clip_improvement": 0.111629 }, "model.layers.2.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.955818, "clip_improvement": 0.111097 }, "model.layers.2.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.20.mlp.down_proj": { "bits": 4, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954335, "clip_improvement": 0.111929 }, "model.layers.20.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954113, "clip_improvement": 0.111808 }, "model.layers.20.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954649, "clip_improvement": 0.111742 }, "model.layers.20.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.20.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954109, "clip_improvement": 0.111827 }, "model.layers.20.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.95533, "clip_improvement": 0.112181 }, "model.layers.20.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.21.mlp.down_proj": { "bits": 4, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954283, "clip_improvement": 0.111674 }, "model.layers.21.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954363, "clip_improvement": 0.11175 }, "model.layers.21.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954025, "clip_improvement": 0.111682 }, "model.layers.21.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.21.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.955643, "clip_improvement": 0.112023 }, "model.layers.21.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.955002, "clip_improvement": 0.111644 }, "model.layers.21.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.22.mlp.down_proj": { "bits": 4, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954025, "clip_improvement": 0.11182 }, "model.layers.22.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954317, "clip_improvement": 0.111939 }, "model.layers.22.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954435, "clip_improvement": 0.111821 }, "model.layers.22.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.22.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954323, "clip_improvement": 0.111696 }, "model.layers.22.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954758, "clip_improvement": 0.111945 }, "model.layers.22.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.23.mlp.down_proj": { "bits": 4, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.953953, "clip_improvement": 0.11189 }, "model.layers.23.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954529, "clip_improvement": 0.111942 }, "model.layers.23.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.953842, "clip_improvement": 0.111662 }, "model.layers.23.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.23.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.953354, "clip_improvement": 0.111375 }, "model.layers.23.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.955055, "clip_improvement": 0.111608 }, "model.layers.23.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.24.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999841, "clip_improvement": 0.250641 }, "model.layers.24.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954001, "clip_improvement": 0.11202 }, "model.layers.24.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.95413, "clip_improvement": 0.111661 }, "model.layers.24.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.952911, "clip_improvement": 0.112427 }, "model.layers.24.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954483, "clip_improvement": 0.111913 }, "model.layers.24.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.953903, "clip_improvement": 0.111511 }, "model.layers.24.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.25.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999882, "clip_improvement": 0.250323 }, "model.layers.25.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954677, "clip_improvement": 0.111964 }, "model.layers.25.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999806, "clip_improvement": 0.250062 }, "model.layers.25.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.955902, "clip_improvement": 0.112661 }, "model.layers.25.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.952873, "clip_improvement": 0.111868 }, "model.layers.25.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.953674, "clip_improvement": 0.11193 }, "model.layers.25.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.26.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999823, "clip_improvement": 0.250825 }, "model.layers.26.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.95459, "clip_improvement": 0.111877 }, "model.layers.26.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.99985, "clip_improvement": 0.250106 }, "model.layers.26.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.26.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.95459, "clip_improvement": 0.112125 }, "model.layers.26.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954407, "clip_improvement": 0.111909 }, "model.layers.26.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.27.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999837, "clip_improvement": 0.250383 }, "model.layers.27.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954333, "clip_improvement": 0.111933 }, "model.layers.27.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999876, "clip_improvement": 0.250137 }, "model.layers.27.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.95462, "clip_improvement": 0.11258 }, "model.layers.27.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.953629, "clip_improvement": 0.111731 }, "model.layers.27.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954422, "clip_improvement": 0.111798 }, "model.layers.27.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.28.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999797, "clip_improvement": 0.25096 }, "model.layers.28.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954136, "clip_improvement": 0.111981 }, "model.layers.28.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.99985, "clip_improvement": 0.250364 }, "model.layers.28.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.28.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.952583, "clip_improvement": 0.11159 }, "model.layers.28.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954071, "clip_improvement": 0.111935 }, "model.layers.28.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.29.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999806, "clip_improvement": 0.250502 }, "model.layers.29.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954518, "clip_improvement": 0.111792 }, "model.layers.29.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999828, "clip_improvement": 0.250467 }, "model.layers.29.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.954102, "clip_improvement": 0.112702 }, "model.layers.29.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.953529, "clip_improvement": 0.111742 }, "model.layers.29.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.955261, "clip_improvement": 0.111977 }, "model.layers.29.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.3.mlp.down_proj": { "bits": 4, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954235, "clip_improvement": 0.111783 }, "model.layers.3.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.95386, "clip_improvement": 0.111778 }, "model.layers.3.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954762, "clip_improvement": 0.111763 }, "model.layers.3.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.3.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.953636, "clip_improvement": 0.111545 }, "model.layers.3.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.955292, "clip_improvement": 0.111747 }, "model.layers.3.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.30.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999786, "clip_improvement": 0.252717 }, "model.layers.30.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954557, "clip_improvement": 0.111757 }, "model.layers.30.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954429, "clip_improvement": 0.11181 }, "model.layers.30.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.955566, "clip_improvement": 0.112461 }, "model.layers.30.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954994, "clip_improvement": 0.111794 }, "model.layers.30.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.953728, "clip_improvement": 0.111865 }, "model.layers.30.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.31.mlp.down_proj": { "bits": 4, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.955824, "clip_improvement": 0.110921 }, "model.layers.31.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954705, "clip_improvement": 0.111719 }, "model.layers.31.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954795, "clip_improvement": 0.111779 }, "model.layers.31.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.31.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.95488, "clip_improvement": 0.112445 }, "model.layers.31.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954491, "clip_improvement": 0.112218 }, "model.layers.31.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.957611, "clip_improvement": 0.112046 }, "model.layers.4.mlp.down_proj": { "bits": 4, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954215, "clip_improvement": 0.111951 }, "model.layers.4.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954629, "clip_improvement": 0.112151 }, "model.layers.4.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954614, "clip_improvement": 0.111802 }, "model.layers.4.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.4.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.953201, "clip_improvement": 0.111462 }, "model.layers.4.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954849, "clip_improvement": 0.111688 }, "model.layers.4.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.5.mlp.down_proj": { "bits": 4, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.953953, "clip_improvement": 0.111808 }, "model.layers.5.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954644, "clip_improvement": 0.111899 }, "model.layers.5.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.955399, "clip_improvement": 0.111783 }, "model.layers.5.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.5.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954559, "clip_improvement": 0.111708 }, "model.layers.5.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.955078, "clip_improvement": 0.111334 }, "model.layers.5.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.6.mlp.down_proj": { "bits": 4, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954808, "clip_improvement": 0.111979 }, "model.layers.6.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.953565, "clip_improvement": 0.111873 }, "model.layers.6.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954544, "clip_improvement": 0.111683 }, "model.layers.6.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.6.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.953156, "clip_improvement": 0.11102 }, "model.layers.6.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.955269, "clip_improvement": 0.112038 }, "model.layers.6.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.7.mlp.down_proj": { "bits": 4, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954546, "clip_improvement": 0.111811 }, "model.layers.7.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954834, "clip_improvement": 0.111859 }, "model.layers.7.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.955078, "clip_improvement": 0.111695 }, "model.layers.7.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.7.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954109, "clip_improvement": 0.111923 }, "model.layers.7.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954811, "clip_improvement": 0.111631 }, "model.layers.7.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.8.mlp.down_proj": { "bits": 4, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954618, "clip_improvement": 0.111939 }, "model.layers.8.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954633, "clip_improvement": 0.111662 }, "model.layers.8.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954692, "clip_improvement": 0.111688 }, "model.layers.8.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.8.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954124, "clip_improvement": 0.111506 }, "model.layers.8.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954231, "clip_improvement": 0.111554 }, "model.layers.8.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.9.mlp.down_proj": { "bits": 4, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954695, "clip_improvement": 0.112018 }, "model.layers.9.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954684, "clip_improvement": 0.111766 }, "model.layers.9.mlp.up_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954535, "clip_improvement": 0.111764 }, "model.layers.9.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.9.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.95401, "clip_improvement": 0.111623 }, "model.layers.9.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954811, "clip_improvement": 0.111774 }, "model.layers.9.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 } } }