{ "schema": "dynquant_checkpoint_v2", "dynquant_core": "0.4.0", "quantization_config": { "quant_method": "dynquant", "checkpoint_format": "dynquant-packed", "schema_version": 1, "version": "0.4.0", "group_size": 128, "symmetric": false, "lm_head_quantized": true, "modules_to_not_convert": [], "modules": { "lm_head": { "bits": 4, "out_features": 32768 }, "model.embed_tokens": { "bits": 3, "out_features": 32768 }, "model.layers.0.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.0.mlp.gate_proj": { "bits": 2, "out_features": 14336 }, "model.layers.0.mlp.up_proj": { "bits": 2, "out_features": 14336 }, "model.layers.0.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.0.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.0.self_attn.q_proj": { "bits": 2, "out_features": 4096 }, "model.layers.0.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.1.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.1.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.1.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.1.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.1.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.1.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.1.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.10.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.10.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.10.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.10.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.10.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.10.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.10.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.11.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.11.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.11.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.11.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.11.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.11.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.11.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.12.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.12.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.12.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.12.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.12.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.12.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.12.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.13.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.13.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.13.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.13.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.13.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.13.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.13.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.14.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.14.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.14.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.14.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.14.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.14.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.14.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.15.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.15.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.15.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.15.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.15.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.15.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.15.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.16.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.16.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.16.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.16.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.16.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.16.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.16.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.17.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.17.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.17.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.17.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.17.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.17.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.17.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.18.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.18.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.18.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.18.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.18.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.18.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.18.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.19.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.19.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.19.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.19.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.19.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.19.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.19.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.2.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.2.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.2.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.2.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.2.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.2.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.2.self_attn.v_proj": { "bits": 8, "out_features": 1024 }, "model.layers.20.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.20.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.20.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.20.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.20.self_attn.o_proj": { "bits": 3, "out_features": 4096 }, "model.layers.20.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.20.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.21.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.21.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.21.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.21.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.21.self_attn.o_proj": { "bits": 3, "out_features": 4096 }, "model.layers.21.self_attn.q_proj": { "bits": 3, "out_features": 4096 }, "model.layers.21.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.22.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.22.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.22.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.22.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.22.self_attn.o_proj": { "bits": 3, "out_features": 4096 }, "model.layers.22.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.22.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.23.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.23.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.23.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.23.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.23.self_attn.o_proj": { "bits": 3, "out_features": 4096 }, "model.layers.23.self_attn.q_proj": { "bits": 3, "out_features": 4096 }, "model.layers.23.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.24.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.24.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.24.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.24.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.24.self_attn.o_proj": { "bits": 3, "out_features": 4096 }, "model.layers.24.self_attn.q_proj": { "bits": 3, "out_features": 4096 }, "model.layers.24.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.25.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.25.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.25.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.25.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.25.self_attn.o_proj": { "bits": 3, "out_features": 4096 }, "model.layers.25.self_attn.q_proj": { "bits": 3, "out_features": 4096 }, "model.layers.25.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.26.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.26.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.26.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.26.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.26.self_attn.o_proj": { "bits": 3, "out_features": 4096 }, "model.layers.26.self_attn.q_proj": { "bits": 3, "out_features": 4096 }, "model.layers.26.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.27.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.27.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.27.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.27.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.27.self_attn.o_proj": { "bits": 2, "out_features": 4096 }, "model.layers.27.self_attn.q_proj": { "bits": 3, "out_features": 4096 }, "model.layers.27.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.28.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.28.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.28.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.28.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.28.self_attn.o_proj": { "bits": 2, "out_features": 4096 }, "model.layers.28.self_attn.q_proj": { "bits": 3, "out_features": 4096 }, "model.layers.28.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.29.mlp.down_proj": { "bits": 2, "out_features": 4096 }, "model.layers.29.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.29.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.29.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.29.self_attn.o_proj": { "bits": 2, "out_features": 4096 }, "model.layers.29.self_attn.q_proj": { "bits": 3, "out_features": 4096 }, "model.layers.29.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.3.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.3.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.3.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.3.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.3.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.3.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.3.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.30.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.30.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.30.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.30.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.30.self_attn.o_proj": { "bits": 3, "out_features": 4096 }, "model.layers.30.self_attn.q_proj": { "bits": 3, "out_features": 4096 }, "model.layers.30.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.31.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.31.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.31.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.31.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.31.self_attn.o_proj": { "bits": 3, "out_features": 4096 }, "model.layers.31.self_attn.q_proj": { "bits": 3, "out_features": 4096 }, "model.layers.31.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.4.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.4.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.4.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.4.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.4.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.4.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.4.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.5.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.5.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.5.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.5.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.5.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.5.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.5.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.6.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.6.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.6.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.6.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.6.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.6.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.6.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.7.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.7.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.7.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.7.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.7.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.7.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.7.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.8.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.8.mlp.gate_proj": { "bits": 4, "out_features": 14336 }, "model.layers.8.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.8.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.8.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.8.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.8.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.9.mlp.down_proj": { "bits": 3, "out_features": 4096 }, "model.layers.9.mlp.gate_proj": { "bits": 3, "out_features": 14336 }, "model.layers.9.mlp.up_proj": { "bits": 3, "out_features": 14336 }, "model.layers.9.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.9.self_attn.o_proj": { "bits": 4, "out_features": 4096 }, "model.layers.9.self_attn.q_proj": { "bits": 4, "out_features": 4096 }, "model.layers.9.self_attn.v_proj": { "bits": 4, "out_features": 1024 } } }, "files": [ "model.safetensors" ], "accounting": { "packed_nbytes": 3067084800, "dense_nbytes": 532480, "total_nbytes": 3067617280, "quantized_elements": 7247757312, "average_bits": 3.385417 }, "tied": {}, "expert_banks": [], "provenance": { "dynquant_core": "0.4.0", "model": "/workspace/runs/s4/mistral7b-v03.text2sql/merged", "stats": null, "allocator": "sensitivity", "widths_from": "/workspace/runs/s4/panel-mistral/maps/dq_3b.json [3068534784]", "predicted_nbytes": 3067617280 }, "layers": { "lm_head": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 32768, 4096 ], "num_rows": 32768, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 71303168, "bits_per_weight": 4.25, "clipped_fraction": 0.953731, "clip_improvement": 0.11396 }, "model.embed_tokens": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 32768, 4096 ], "num_rows": 32768, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 54525952, "bits_per_weight": 3.25, "clipped_fraction": 0.972293, "clip_improvement": 0.250255 }, "model.layers.0.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.99985, "clip_improvement": 0.250922 }, "model.layers.0.mlp.gate_proj": { "bits": 2, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16515072, "bits_per_weight": 2.25, "clipped_fraction": 0.999948, "clip_improvement": 0.353799 }, "model.layers.0.mlp.up_proj": { "bits": 2, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16515072, "bits_per_weight": 2.25, "clipped_fraction": 0.999948, "clip_improvement": 0.354368 }, "model.layers.0.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.89209, "clip_improvement": 0.21042 }, "model.layers.0.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.955269, "clip_improvement": 0.111095 }, "model.layers.0.self_attn.q_proj": { "bits": 2, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4718592, "bits_per_weight": 2.25, "clipped_fraction": 0.944618, "clip_improvement": 0.403019 }, "model.layers.0.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.968658, "clip_improvement": 0.106915 }, "model.layers.1.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999832, "clip_improvement": 0.253984 }, "model.layers.1.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999839, "clip_improvement": 0.250635 }, "model.layers.1.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.99976, "clip_improvement": 0.250698 }, "model.layers.1.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.915283, "clip_improvement": 0.146744 }, "model.layers.1.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954872, "clip_improvement": 0.111292 }, "model.layers.1.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.926826, "clip_improvement": 0.134538 }, "model.layers.1.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.959137, "clip_improvement": 0.109709 }, "model.layers.10.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999832, "clip_improvement": 0.251603 }, "model.layers.10.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.99981, "clip_improvement": 0.251404 }, "model.layers.10.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999826, "clip_improvement": 0.250946 }, "model.layers.10.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.954437, "clip_improvement": 0.11216 }, "model.layers.10.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954659, "clip_improvement": 0.111894 }, "model.layers.10.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.955727, "clip_improvement": 0.111726 }, "model.layers.10.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.953369, "clip_improvement": 0.111091 }, "model.layers.11.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999776, "clip_improvement": 0.251633 }, "model.layers.11.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999806, "clip_improvement": 0.251636 }, "model.layers.11.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999834, "clip_improvement": 0.251204 }, "model.layers.11.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.953491, "clip_improvement": 0.111377 }, "model.layers.11.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954651, "clip_improvement": 0.111641 }, "model.layers.11.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.9543, "clip_improvement": 0.11162 }, "model.layers.11.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.954468, "clip_improvement": 0.111403 }, "model.layers.12.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999823, "clip_improvement": 0.250881 }, "model.layers.12.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.99983, "clip_improvement": 0.251423 }, "model.layers.12.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999834, "clip_improvement": 0.250581 }, "model.layers.12.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.954712, "clip_improvement": 0.111661 }, "model.layers.12.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954231, "clip_improvement": 0.111711 }, "model.layers.12.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954811, "clip_improvement": 0.110958 }, "model.layers.12.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.957977, "clip_improvement": 0.111401 }, "model.layers.13.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999795, "clip_improvement": 0.251866 }, "model.layers.13.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999799, "clip_improvement": 0.251658 }, "model.layers.13.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999847, "clip_improvement": 0.250763 }, "model.layers.13.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.955353, "clip_improvement": 0.112327 }, "model.layers.13.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954491, "clip_improvement": 0.11161 }, "model.layers.13.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.955147, "clip_improvement": 0.111867 }, "model.layers.13.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.955048, "clip_improvement": 0.110047 }, "model.layers.14.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999808, "clip_improvement": 0.25274 }, "model.layers.14.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.955198, "clip_improvement": 0.11176 }, "model.layers.14.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999861, "clip_improvement": 0.250739 }, "model.layers.14.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.955048, "clip_improvement": 0.112272 }, "model.layers.14.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954231, "clip_improvement": 0.111575 }, "model.layers.14.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.95491, "clip_improvement": 0.111332 }, "model.layers.14.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.954773, "clip_improvement": 0.110612 }, "model.layers.15.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999837, "clip_improvement": 0.251755 }, "model.layers.15.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999832, "clip_improvement": 0.251587 }, "model.layers.15.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999861, "clip_improvement": 0.2507 }, "model.layers.15.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.954468, "clip_improvement": 0.111725 }, "model.layers.15.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954422, "clip_improvement": 0.11157 }, "model.layers.15.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954803, "clip_improvement": 0.111575 }, "model.layers.15.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.953522, "clip_improvement": 0.110272 }, "model.layers.16.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999808, "clip_improvement": 0.251313 }, "model.layers.16.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999804, "clip_improvement": 0.251691 }, "model.layers.16.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999861, "clip_improvement": 0.250844 }, "model.layers.16.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.953979, "clip_improvement": 0.111986 }, "model.layers.16.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954048, "clip_improvement": 0.111673 }, "model.layers.16.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954651, "clip_improvement": 0.111957 }, "model.layers.16.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.955353, "clip_improvement": 0.11089 }, "model.layers.17.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.99978, "clip_improvement": 0.252067 }, "model.layers.17.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999832, "clip_improvement": 0.251504 }, "model.layers.17.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999858, "clip_improvement": 0.250843 }, "model.layers.17.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.955719, "clip_improvement": 0.111792 }, "model.layers.17.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.95369, "clip_improvement": 0.111864 }, "model.layers.17.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954697, "clip_improvement": 0.111357 }, "model.layers.17.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.955933, "clip_improvement": 0.110902 }, "model.layers.18.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999795, "clip_improvement": 0.251975 }, "model.layers.18.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999841, "clip_improvement": 0.251286 }, "model.layers.18.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999845, "clip_improvement": 0.250551 }, "model.layers.18.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.95462, "clip_improvement": 0.111076 }, "model.layers.18.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.953995, "clip_improvement": 0.111652 }, "model.layers.18.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.955261, "clip_improvement": 0.111183 }, "model.layers.18.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.957062, "clip_improvement": 0.111033 }, "model.layers.19.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999802, "clip_improvement": 0.252936 }, "model.layers.19.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.99983, "clip_improvement": 0.250944 }, "model.layers.19.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999841, "clip_improvement": 0.250607 }, "model.layers.19.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.957581, "clip_improvement": 0.112968 }, "model.layers.19.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.95388, "clip_improvement": 0.112022 }, "model.layers.19.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.953545, "clip_improvement": 0.111476 }, "model.layers.19.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.955017, "clip_improvement": 0.111189 }, "model.layers.2.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999813, "clip_improvement": 0.2507 }, "model.layers.2.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999845, "clip_improvement": 0.250663 }, "model.layers.2.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999867, "clip_improvement": 0.250393 }, "model.layers.2.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.956146, "clip_improvement": 0.111314 }, "model.layers.2.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954819, "clip_improvement": 0.111629 }, "model.layers.2.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.955818, "clip_improvement": 0.111097 }, "model.layers.2.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4325376, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.20.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999819, "clip_improvement": 0.251106 }, "model.layers.20.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999832, "clip_improvement": 0.250708 }, "model.layers.20.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999841, "clip_improvement": 0.250405 }, "model.layers.20.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.956512, "clip_improvement": 0.111956 }, "model.layers.20.self_attn.o_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6815744, "bits_per_weight": 3.25, "clipped_fraction": 0.999863, "clip_improvement": 0.249715 }, "model.layers.20.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.95533, "clip_improvement": 0.112181 }, "model.layers.20.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.955841, "clip_improvement": 0.111293 }, "model.layers.21.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999843, "clip_improvement": 0.250926 }, "model.layers.21.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999834, "clip_improvement": 0.250467 }, "model.layers.21.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999852, "clip_improvement": 0.25041 }, "model.layers.21.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.952942, "clip_improvement": 0.112039 }, "model.layers.21.self_attn.o_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6815744, "bits_per_weight": 3.25, "clipped_fraction": 0.999863, "clip_improvement": 0.249658 }, "model.layers.21.self_attn.q_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6815744, "bits_per_weight": 3.25, "clipped_fraction": 0.999817, "clip_improvement": 0.250514 }, "model.layers.21.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.955994, "clip_improvement": 0.110923 }, "model.layers.22.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999856, "clip_improvement": 0.250694 }, "model.layers.22.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999843, "clip_improvement": 0.25018 }, "model.layers.22.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999823, "clip_improvement": 0.250299 }, "model.layers.22.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.953308, "clip_improvement": 0.112909 }, "model.layers.22.self_attn.o_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6815744, "bits_per_weight": 3.25, "clipped_fraction": 0.99987, "clip_improvement": 0.250194 }, "model.layers.22.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954758, "clip_improvement": 0.111945 }, "model.layers.22.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.955688, "clip_improvement": 0.111754 }, "model.layers.23.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999828, "clip_improvement": 0.250656 }, "model.layers.23.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999878, "clip_improvement": 0.250288 }, "model.layers.23.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999867, "clip_improvement": 0.250226 }, "model.layers.23.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.954498, "clip_improvement": 0.111951 }, "model.layers.23.self_attn.o_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6815744, "bits_per_weight": 3.25, "clipped_fraction": 0.999855, "clip_improvement": 0.249696 }, "model.layers.23.self_attn.q_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6815744, "bits_per_weight": 3.25, "clipped_fraction": 0.999817, "clip_improvement": 0.25049 }, "model.layers.23.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.955231, "clip_improvement": 0.111331 }, "model.layers.24.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999841, "clip_improvement": 0.250641 }, "model.layers.24.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999847, "clip_improvement": 0.25055 }, "model.layers.24.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999802, "clip_improvement": 0.250355 }, "model.layers.24.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.952911, "clip_improvement": 0.112427 }, "model.layers.24.self_attn.o_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6815744, "bits_per_weight": 3.25, "clipped_fraction": 0.999893, "clip_improvement": 0.249678 }, "model.layers.24.self_attn.q_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6815744, "bits_per_weight": 3.25, "clipped_fraction": 0.999763, "clip_improvement": 0.250684 }, "model.layers.24.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.955963, "clip_improvement": 0.112094 }, "model.layers.25.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999882, "clip_improvement": 0.250323 }, "model.layers.25.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999815, "clip_improvement": 0.250262 }, "model.layers.25.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999806, "clip_improvement": 0.250062 }, "model.layers.25.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.955902, "clip_improvement": 0.112661 }, "model.layers.25.self_attn.o_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6815744, "bits_per_weight": 3.25, "clipped_fraction": 0.999886, "clip_improvement": 0.25052 }, "model.layers.25.self_attn.q_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6815744, "bits_per_weight": 3.25, "clipped_fraction": 0.999855, "clip_improvement": 0.250996 }, "model.layers.25.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.955231, "clip_improvement": 0.111858 }, "model.layers.26.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999823, "clip_improvement": 0.250825 }, "model.layers.26.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.99983, "clip_improvement": 0.250064 }, "model.layers.26.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.99985, "clip_improvement": 0.250106 }, "model.layers.26.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.95578, "clip_improvement": 0.112435 }, "model.layers.26.self_attn.o_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6815744, "bits_per_weight": 3.25, "clipped_fraction": 0.999832, "clip_improvement": 0.250295 }, "model.layers.26.self_attn.q_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6815744, "bits_per_weight": 3.25, "clipped_fraction": 0.999916, "clip_improvement": 0.250897 }, "model.layers.26.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.956543, "clip_improvement": 0.112278 }, "model.layers.27.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999837, "clip_improvement": 0.250383 }, "model.layers.27.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999837, "clip_improvement": 0.250377 }, "model.layers.27.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999876, "clip_improvement": 0.250137 }, "model.layers.27.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.95462, "clip_improvement": 0.11258 }, "model.layers.27.self_attn.o_proj": { "bits": 2, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4718592, "bits_per_weight": 2.25, "clipped_fraction": 1.0, "clip_improvement": 0.353649 }, "model.layers.27.self_attn.q_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6815744, "bits_per_weight": 3.25, "clipped_fraction": 0.999878, "clip_improvement": 0.251101 }, "model.layers.27.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.954529, "clip_improvement": 0.111168 }, "model.layers.28.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999797, "clip_improvement": 0.25096 }, "model.layers.28.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999837, "clip_improvement": 0.250603 }, "model.layers.28.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.99985, "clip_improvement": 0.250364 }, "model.layers.28.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.954224, "clip_improvement": 0.112371 }, "model.layers.28.self_attn.o_proj": { "bits": 2, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4718592, "bits_per_weight": 2.25, "clipped_fraction": 0.999977, "clip_improvement": 0.353555 }, "model.layers.28.self_attn.q_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6815744, "bits_per_weight": 3.25, "clipped_fraction": 0.99987, "clip_improvement": 0.251118 }, "model.layers.28.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.954681, "clip_improvement": 0.110762 }, "model.layers.29.mlp.down_proj": { "bits": 2, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16515072, "bits_per_weight": 2.25, "clipped_fraction": 0.999867, "clip_improvement": 0.354033 }, "model.layers.29.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999813, "clip_improvement": 0.250075 }, "model.layers.29.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999828, "clip_improvement": 0.250467 }, "model.layers.29.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.954102, "clip_improvement": 0.112702 }, "model.layers.29.self_attn.o_proj": { "bits": 2, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 4718592, "bits_per_weight": 2.25, "clipped_fraction": 1.0, "clip_improvement": 0.353919 }, "model.layers.29.self_attn.q_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6815744, "bits_per_weight": 3.25, "clipped_fraction": 0.999855, "clip_improvement": 0.252049 }, "model.layers.29.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.955322, "clip_improvement": 0.110842 }, "model.layers.3.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999823, "clip_improvement": 0.250621 }, "model.layers.3.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999808, "clip_improvement": 0.250524 }, "model.layers.3.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999839, "clip_improvement": 0.250106 }, "model.layers.3.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.956268, "clip_improvement": 0.111218 }, "model.layers.3.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.953636, "clip_improvement": 0.111545 }, "model.layers.3.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.955292, "clip_improvement": 0.111747 }, "model.layers.3.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.953796, "clip_improvement": 0.111013 }, "model.layers.30.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999786, "clip_improvement": 0.252717 }, "model.layers.30.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999823, "clip_improvement": 0.250554 }, "model.layers.30.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.99983, "clip_improvement": 0.250294 }, "model.layers.30.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.955566, "clip_improvement": 0.112461 }, "model.layers.30.self_attn.o_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6815744, "bits_per_weight": 3.25, "clipped_fraction": 0.999802, "clip_improvement": 0.250114 }, "model.layers.30.self_attn.q_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6815744, "bits_per_weight": 3.25, "clipped_fraction": 0.999825, "clip_improvement": 0.251371 }, "model.layers.30.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.959351, "clip_improvement": 0.111455 }, "model.layers.31.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999821, "clip_improvement": 0.252817 }, "model.layers.31.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.250397 }, "model.layers.31.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999878, "clip_improvement": 0.250394 }, "model.layers.31.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.954468, "clip_improvement": 0.111969 }, "model.layers.31.self_attn.o_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6815744, "bits_per_weight": 3.25, "clipped_fraction": 0.999825, "clip_improvement": 0.252438 }, "model.layers.31.self_attn.q_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 6815744, "bits_per_weight": 3.25, "clipped_fraction": 0.999802, "clip_improvement": 0.251303 }, "model.layers.31.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.957611, "clip_improvement": 0.112046 }, "model.layers.4.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999821, "clip_improvement": 0.251359 }, "model.layers.4.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999797, "clip_improvement": 0.251122 }, "model.layers.4.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999852, "clip_improvement": 0.250753 }, "model.layers.4.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.95459, "clip_improvement": 0.112584 }, "model.layers.4.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.953201, "clip_improvement": 0.111462 }, "model.layers.4.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954849, "clip_improvement": 0.111688 }, "model.layers.4.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.953674, "clip_improvement": 0.111637 }, "model.layers.5.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999747, "clip_improvement": 0.250899 }, "model.layers.5.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999747, "clip_improvement": 0.251304 }, "model.layers.5.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999813, "clip_improvement": 0.250499 }, "model.layers.5.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.956085, "clip_improvement": 0.1119 }, "model.layers.5.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954559, "clip_improvement": 0.111708 }, "model.layers.5.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.955078, "clip_improvement": 0.111334 }, "model.layers.5.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.953003, "clip_improvement": 0.111743 }, "model.layers.6.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999797, "clip_improvement": 0.251373 }, "model.layers.6.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999808, "clip_improvement": 0.250786 }, "model.layers.6.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999815, "clip_improvement": 0.250891 }, "model.layers.6.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.955719, "clip_improvement": 0.11243 }, "model.layers.6.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.953156, "clip_improvement": 0.11102 }, "model.layers.6.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.955269, "clip_improvement": 0.112038 }, "model.layers.6.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.955841, "clip_improvement": 0.111308 }, "model.layers.7.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999802, "clip_improvement": 0.251257 }, "model.layers.7.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999819, "clip_improvement": 0.250713 }, "model.layers.7.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999843, "clip_improvement": 0.250846 }, "model.layers.7.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.954742, "clip_improvement": 0.111861 }, "model.layers.7.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954109, "clip_improvement": 0.111923 }, "model.layers.7.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954811, "clip_improvement": 0.111631 }, "model.layers.7.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.955536, "clip_improvement": 0.11122 }, "model.layers.8.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999786, "clip_improvement": 0.250944 }, "model.layers.8.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 31195136, "bits_per_weight": 4.25, "clipped_fraction": 0.954633, "clip_improvement": 0.111662 }, "model.layers.8.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.99983, "clip_improvement": 0.250718 }, "model.layers.8.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.953888, "clip_improvement": 0.111444 }, "model.layers.8.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954124, "clip_improvement": 0.111506 }, "model.layers.8.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954231, "clip_improvement": 0.111554 }, "model.layers.8.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.953949, "clip_improvement": 0.111854 }, "model.layers.9.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 14336, "logical_shape": [ 4096, 14336 ], "num_rows": 4096, "num_groups": 112, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999869, "clip_improvement": 0.250692 }, "model.layers.9.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999804, "clip_improvement": 0.250961 }, "model.layers.9.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 4096, "logical_shape": [ 14336, 4096 ], "num_rows": 14336, "num_groups": 32, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 23855104, "bits_per_weight": 3.25, "clipped_fraction": 0.999865, "clip_improvement": 0.250806 }, "model.layers.9.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.953949, "clip_improvement": 0.111844 }, "model.layers.9.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.95401, "clip_improvement": 0.111623 }, "model.layers.9.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 4096, 4096 ], "num_rows": 4096, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8912896, "bits_per_weight": 4.25, "clipped_fraction": 0.954811, "clip_improvement": 0.111774 }, "model.layers.9.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 4096, "logical_shape": [ 1024, 4096 ], "num_rows": 1024, "num_groups": 32, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2228224, "bits_per_weight": 4.25, "clipped_fraction": 0.954956, "clip_improvement": 0.110794 } } }