| { |
| "schema": "dynquant_checkpoint_v2", |
| "dynquant_core": "0.4.0", |
| "quantization_config": { |
| "quant_method": "dynquant", |
| "checkpoint_format": "dynquant-packed", |
| "schema_version": 1, |
| "version": "0.4.0", |
| "group_size": 128, |
| "symmetric": false, |
| "lm_head_quantized": true, |
| "modules_to_not_convert": [], |
| "modules": { |
| "lm_head": { |
| "bits": 4, |
| "out_features": 32768 |
| }, |
| "model.embed_tokens": { |
| "bits": 3, |
| "out_features": 32768 |
| }, |
| "model.layers.0.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.0.mlp.gate_proj": { |
| "bits": 2, |
| "out_features": 14336 |
| }, |
| "model.layers.0.mlp.up_proj": { |
| "bits": 2, |
| "out_features": 14336 |
| }, |
| "model.layers.0.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.0.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.0.self_attn.q_proj": { |
| "bits": 2, |
| "out_features": 4096 |
| }, |
| "model.layers.0.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.1.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.1.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.1.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.1.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.1.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.1.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.1.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.10.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.10.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.10.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.10.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.10.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.10.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.10.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.11.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.11.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.11.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.11.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.11.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.11.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.11.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.12.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.12.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.12.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.12.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.12.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.12.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.12.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.13.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.13.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.13.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.13.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.13.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.13.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.13.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.14.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.14.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.14.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.14.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.14.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.14.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.14.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.15.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.15.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.15.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.15.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.15.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.15.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.15.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.16.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.16.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.16.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.16.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.16.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.16.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.16.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.17.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.17.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.17.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.17.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.17.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.17.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.17.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.18.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.18.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.18.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.18.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.18.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.18.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.18.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.19.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.19.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.19.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.19.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.19.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.19.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.19.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.2.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.2.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.2.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.2.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.2.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.2.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.2.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.20.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.20.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.20.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.20.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.20.self_attn.o_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.20.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.20.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.21.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.21.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.21.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.21.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.21.self_attn.o_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.21.self_attn.q_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.21.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.22.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.22.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.22.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.22.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.22.self_attn.o_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.22.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.22.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.23.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.23.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.23.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.23.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.23.self_attn.o_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.23.self_attn.q_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.23.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.24.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.24.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.24.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.24.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.24.self_attn.o_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.24.self_attn.q_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.24.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.25.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.25.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.25.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.25.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.25.self_attn.o_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.25.self_attn.q_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.25.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.26.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.26.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.26.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.26.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.26.self_attn.o_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.26.self_attn.q_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.26.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.27.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.27.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.27.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.27.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.27.self_attn.o_proj": { |
| "bits": 2, |
| "out_features": 4096 |
| }, |
| "model.layers.27.self_attn.q_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.27.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.28.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.28.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.28.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.28.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.28.self_attn.o_proj": { |
| "bits": 2, |
| "out_features": 4096 |
| }, |
| "model.layers.28.self_attn.q_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.28.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.29.mlp.down_proj": { |
| "bits": 2, |
| "out_features": 4096 |
| }, |
| "model.layers.29.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.29.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.29.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.29.self_attn.o_proj": { |
| "bits": 2, |
| "out_features": 4096 |
| }, |
| "model.layers.29.self_attn.q_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.29.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.3.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.3.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.3.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.3.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.3.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.3.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.3.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.30.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.30.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.30.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.30.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.30.self_attn.o_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.30.self_attn.q_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.30.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.31.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.31.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.31.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.31.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.31.self_attn.o_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.31.self_attn.q_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.31.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.4.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.4.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.4.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.4.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.4.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.4.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.4.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.5.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.5.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.5.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.5.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.5.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.5.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.5.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.6.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.6.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.6.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.6.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.6.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.6.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.6.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.7.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.7.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.7.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.7.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.7.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.7.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.7.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.8.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.8.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.8.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.8.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.8.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.8.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.8.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.9.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.9.mlp.gate_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.9.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.9.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.9.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.9.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.9.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| } |
| } |
| }, |
| "files": [ |
| "model.safetensors" |
| ], |
| "accounting": { |
| "packed_nbytes": 3067084800, |
| "dense_nbytes": 532480, |
| "total_nbytes": 3067617280, |
| "quantized_elements": 7247757312, |
| "average_bits": 3.385417 |
| }, |
| "tied": {}, |
| "expert_banks": [], |
| "provenance": { |
| "dynquant_core": "0.4.0", |
| "model": "/workspace/runs/s4/mistral7b-v03.text2sql/merged", |
| "stats": null, |
| "allocator": "sensitivity", |
| "widths_from": "/workspace/runs/s4/panel-mistral/maps/dq_3b.json [3068534784]", |
| "predicted_nbytes": 3067617280 |
| }, |
| "layers": { |
| "lm_head": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 32768, |
| 4096 |
| ], |
| "num_rows": 32768, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 71303168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953731, |
| "clip_improvement": 0.11396 |
| }, |
| "model.embed_tokens": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 32768, |
| 4096 |
| ], |
| "num_rows": 32768, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 54525952, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.972293, |
| "clip_improvement": 0.250255 |
| }, |
| "model.layers.0.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.99985, |
| "clip_improvement": 0.250922 |
| }, |
| "model.layers.0.mlp.gate_proj": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 16515072, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999948, |
| "clip_improvement": 0.353799 |
| }, |
| "model.layers.0.mlp.up_proj": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 16515072, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999948, |
| "clip_improvement": 0.354368 |
| }, |
| "model.layers.0.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.89209, |
| "clip_improvement": 0.21042 |
| }, |
| "model.layers.0.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955269, |
| "clip_improvement": 0.111095 |
| }, |
| "model.layers.0.self_attn.q_proj": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4718592, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.944618, |
| "clip_improvement": 0.403019 |
| }, |
| "model.layers.0.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.968658, |
| "clip_improvement": 0.106915 |
| }, |
| "model.layers.1.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999832, |
| "clip_improvement": 0.253984 |
| }, |
| "model.layers.1.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999839, |
| "clip_improvement": 0.250635 |
| }, |
| "model.layers.1.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.99976, |
| "clip_improvement": 0.250698 |
| }, |
| "model.layers.1.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.915283, |
| "clip_improvement": 0.146744 |
| }, |
| "model.layers.1.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954872, |
| "clip_improvement": 0.111292 |
| }, |
| "model.layers.1.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.926826, |
| "clip_improvement": 0.134538 |
| }, |
| "model.layers.1.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.959137, |
| "clip_improvement": 0.109709 |
| }, |
| "model.layers.10.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999832, |
| "clip_improvement": 0.251603 |
| }, |
| "model.layers.10.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.99981, |
| "clip_improvement": 0.251404 |
| }, |
| "model.layers.10.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999826, |
| "clip_improvement": 0.250946 |
| }, |
| "model.layers.10.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954437, |
| "clip_improvement": 0.11216 |
| }, |
| "model.layers.10.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954659, |
| "clip_improvement": 0.111894 |
| }, |
| "model.layers.10.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955727, |
| "clip_improvement": 0.111726 |
| }, |
| "model.layers.10.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953369, |
| "clip_improvement": 0.111091 |
| }, |
| "model.layers.11.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999776, |
| "clip_improvement": 0.251633 |
| }, |
| "model.layers.11.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999806, |
| "clip_improvement": 0.251636 |
| }, |
| "model.layers.11.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999834, |
| "clip_improvement": 0.251204 |
| }, |
| "model.layers.11.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953491, |
| "clip_improvement": 0.111377 |
| }, |
| "model.layers.11.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954651, |
| "clip_improvement": 0.111641 |
| }, |
| "model.layers.11.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.9543, |
| "clip_improvement": 0.11162 |
| }, |
| "model.layers.11.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954468, |
| "clip_improvement": 0.111403 |
| }, |
| "model.layers.12.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999823, |
| "clip_improvement": 0.250881 |
| }, |
| "model.layers.12.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.99983, |
| "clip_improvement": 0.251423 |
| }, |
| "model.layers.12.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999834, |
| "clip_improvement": 0.250581 |
| }, |
| "model.layers.12.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954712, |
| "clip_improvement": 0.111661 |
| }, |
| "model.layers.12.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954231, |
| "clip_improvement": 0.111711 |
| }, |
| "model.layers.12.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954811, |
| "clip_improvement": 0.110958 |
| }, |
| "model.layers.12.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.957977, |
| "clip_improvement": 0.111401 |
| }, |
| "model.layers.13.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999795, |
| "clip_improvement": 0.251866 |
| }, |
| "model.layers.13.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999799, |
| "clip_improvement": 0.251658 |
| }, |
| "model.layers.13.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999847, |
| "clip_improvement": 0.250763 |
| }, |
| "model.layers.13.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955353, |
| "clip_improvement": 0.112327 |
| }, |
| "model.layers.13.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954491, |
| "clip_improvement": 0.11161 |
| }, |
| "model.layers.13.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955147, |
| "clip_improvement": 0.111867 |
| }, |
| "model.layers.13.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955048, |
| "clip_improvement": 0.110047 |
| }, |
| "model.layers.14.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999808, |
| "clip_improvement": 0.25274 |
| }, |
| "model.layers.14.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955198, |
| "clip_improvement": 0.11176 |
| }, |
| "model.layers.14.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999861, |
| "clip_improvement": 0.250739 |
| }, |
| "model.layers.14.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955048, |
| "clip_improvement": 0.112272 |
| }, |
| "model.layers.14.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954231, |
| "clip_improvement": 0.111575 |
| }, |
| "model.layers.14.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95491, |
| "clip_improvement": 0.111332 |
| }, |
| "model.layers.14.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954773, |
| "clip_improvement": 0.110612 |
| }, |
| "model.layers.15.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999837, |
| "clip_improvement": 0.251755 |
| }, |
| "model.layers.15.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999832, |
| "clip_improvement": 0.251587 |
| }, |
| "model.layers.15.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999861, |
| "clip_improvement": 0.2507 |
| }, |
| "model.layers.15.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954468, |
| "clip_improvement": 0.111725 |
| }, |
| "model.layers.15.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954422, |
| "clip_improvement": 0.11157 |
| }, |
| "model.layers.15.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954803, |
| "clip_improvement": 0.111575 |
| }, |
| "model.layers.15.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953522, |
| "clip_improvement": 0.110272 |
| }, |
| "model.layers.16.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999808, |
| "clip_improvement": 0.251313 |
| }, |
| "model.layers.16.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999804, |
| "clip_improvement": 0.251691 |
| }, |
| "model.layers.16.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999861, |
| "clip_improvement": 0.250844 |
| }, |
| "model.layers.16.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953979, |
| "clip_improvement": 0.111986 |
| }, |
| "model.layers.16.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954048, |
| "clip_improvement": 0.111673 |
| }, |
| "model.layers.16.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954651, |
| "clip_improvement": 0.111957 |
| }, |
| "model.layers.16.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955353, |
| "clip_improvement": 0.11089 |
| }, |
| "model.layers.17.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.99978, |
| "clip_improvement": 0.252067 |
| }, |
| "model.layers.17.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999832, |
| "clip_improvement": 0.251504 |
| }, |
| "model.layers.17.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999858, |
| "clip_improvement": 0.250843 |
| }, |
| "model.layers.17.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955719, |
| "clip_improvement": 0.111792 |
| }, |
| "model.layers.17.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95369, |
| "clip_improvement": 0.111864 |
| }, |
| "model.layers.17.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954697, |
| "clip_improvement": 0.111357 |
| }, |
| "model.layers.17.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955933, |
| "clip_improvement": 0.110902 |
| }, |
| "model.layers.18.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999795, |
| "clip_improvement": 0.251975 |
| }, |
| "model.layers.18.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999841, |
| "clip_improvement": 0.251286 |
| }, |
| "model.layers.18.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999845, |
| "clip_improvement": 0.250551 |
| }, |
| "model.layers.18.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95462, |
| "clip_improvement": 0.111076 |
| }, |
| "model.layers.18.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953995, |
| "clip_improvement": 0.111652 |
| }, |
| "model.layers.18.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955261, |
| "clip_improvement": 0.111183 |
| }, |
| "model.layers.18.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.957062, |
| "clip_improvement": 0.111033 |
| }, |
| "model.layers.19.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999802, |
| "clip_improvement": 0.252936 |
| }, |
| "model.layers.19.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.99983, |
| "clip_improvement": 0.250944 |
| }, |
| "model.layers.19.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999841, |
| "clip_improvement": 0.250607 |
| }, |
| "model.layers.19.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.957581, |
| "clip_improvement": 0.112968 |
| }, |
| "model.layers.19.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95388, |
| "clip_improvement": 0.112022 |
| }, |
| "model.layers.19.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953545, |
| "clip_improvement": 0.111476 |
| }, |
| "model.layers.19.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955017, |
| "clip_improvement": 0.111189 |
| }, |
| "model.layers.2.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999813, |
| "clip_improvement": 0.2507 |
| }, |
| "model.layers.2.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999845, |
| "clip_improvement": 0.250663 |
| }, |
| "model.layers.2.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999867, |
| "clip_improvement": 0.250393 |
| }, |
| "model.layers.2.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.956146, |
| "clip_improvement": 0.111314 |
| }, |
| "model.layers.2.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954819, |
| "clip_improvement": 0.111629 |
| }, |
| "model.layers.2.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955818, |
| "clip_improvement": 0.111097 |
| }, |
| "model.layers.2.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.20.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999819, |
| "clip_improvement": 0.251106 |
| }, |
| "model.layers.20.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999832, |
| "clip_improvement": 0.250708 |
| }, |
| "model.layers.20.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999841, |
| "clip_improvement": 0.250405 |
| }, |
| "model.layers.20.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.956512, |
| "clip_improvement": 0.111956 |
| }, |
| "model.layers.20.self_attn.o_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6815744, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999863, |
| "clip_improvement": 0.249715 |
| }, |
| "model.layers.20.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95533, |
| "clip_improvement": 0.112181 |
| }, |
| "model.layers.20.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955841, |
| "clip_improvement": 0.111293 |
| }, |
| "model.layers.21.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999843, |
| "clip_improvement": 0.250926 |
| }, |
| "model.layers.21.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999834, |
| "clip_improvement": 0.250467 |
| }, |
| "model.layers.21.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999852, |
| "clip_improvement": 0.25041 |
| }, |
| "model.layers.21.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952942, |
| "clip_improvement": 0.112039 |
| }, |
| "model.layers.21.self_attn.o_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6815744, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999863, |
| "clip_improvement": 0.249658 |
| }, |
| "model.layers.21.self_attn.q_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6815744, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999817, |
| "clip_improvement": 0.250514 |
| }, |
| "model.layers.21.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955994, |
| "clip_improvement": 0.110923 |
| }, |
| "model.layers.22.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999856, |
| "clip_improvement": 0.250694 |
| }, |
| "model.layers.22.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999843, |
| "clip_improvement": 0.25018 |
| }, |
| "model.layers.22.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999823, |
| "clip_improvement": 0.250299 |
| }, |
| "model.layers.22.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953308, |
| "clip_improvement": 0.112909 |
| }, |
| "model.layers.22.self_attn.o_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6815744, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.99987, |
| "clip_improvement": 0.250194 |
| }, |
| "model.layers.22.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954758, |
| "clip_improvement": 0.111945 |
| }, |
| "model.layers.22.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955688, |
| "clip_improvement": 0.111754 |
| }, |
| "model.layers.23.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999828, |
| "clip_improvement": 0.250656 |
| }, |
| "model.layers.23.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999878, |
| "clip_improvement": 0.250288 |
| }, |
| "model.layers.23.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999867, |
| "clip_improvement": 0.250226 |
| }, |
| "model.layers.23.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954498, |
| "clip_improvement": 0.111951 |
| }, |
| "model.layers.23.self_attn.o_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6815744, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999855, |
| "clip_improvement": 0.249696 |
| }, |
| "model.layers.23.self_attn.q_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6815744, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999817, |
| "clip_improvement": 0.25049 |
| }, |
| "model.layers.23.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955231, |
| "clip_improvement": 0.111331 |
| }, |
| "model.layers.24.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999841, |
| "clip_improvement": 0.250641 |
| }, |
| "model.layers.24.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999847, |
| "clip_improvement": 0.25055 |
| }, |
| "model.layers.24.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999802, |
| "clip_improvement": 0.250355 |
| }, |
| "model.layers.24.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952911, |
| "clip_improvement": 0.112427 |
| }, |
| "model.layers.24.self_attn.o_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6815744, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999893, |
| "clip_improvement": 0.249678 |
| }, |
| "model.layers.24.self_attn.q_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6815744, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999763, |
| "clip_improvement": 0.250684 |
| }, |
| "model.layers.24.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955963, |
| "clip_improvement": 0.112094 |
| }, |
| "model.layers.25.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999882, |
| "clip_improvement": 0.250323 |
| }, |
| "model.layers.25.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999815, |
| "clip_improvement": 0.250262 |
| }, |
| "model.layers.25.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999806, |
| "clip_improvement": 0.250062 |
| }, |
| "model.layers.25.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955902, |
| "clip_improvement": 0.112661 |
| }, |
| "model.layers.25.self_attn.o_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6815744, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999886, |
| "clip_improvement": 0.25052 |
| }, |
| "model.layers.25.self_attn.q_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6815744, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999855, |
| "clip_improvement": 0.250996 |
| }, |
| "model.layers.25.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955231, |
| "clip_improvement": 0.111858 |
| }, |
| "model.layers.26.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999823, |
| "clip_improvement": 0.250825 |
| }, |
| "model.layers.26.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.99983, |
| "clip_improvement": 0.250064 |
| }, |
| "model.layers.26.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.99985, |
| "clip_improvement": 0.250106 |
| }, |
| "model.layers.26.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95578, |
| "clip_improvement": 0.112435 |
| }, |
| "model.layers.26.self_attn.o_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6815744, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999832, |
| "clip_improvement": 0.250295 |
| }, |
| "model.layers.26.self_attn.q_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6815744, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999916, |
| "clip_improvement": 0.250897 |
| }, |
| "model.layers.26.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.956543, |
| "clip_improvement": 0.112278 |
| }, |
| "model.layers.27.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999837, |
| "clip_improvement": 0.250383 |
| }, |
| "model.layers.27.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999837, |
| "clip_improvement": 0.250377 |
| }, |
| "model.layers.27.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999876, |
| "clip_improvement": 0.250137 |
| }, |
| "model.layers.27.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95462, |
| "clip_improvement": 0.11258 |
| }, |
| "model.layers.27.self_attn.o_proj": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4718592, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 1.0, |
| "clip_improvement": 0.353649 |
| }, |
| "model.layers.27.self_attn.q_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6815744, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999878, |
| "clip_improvement": 0.251101 |
| }, |
| "model.layers.27.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954529, |
| "clip_improvement": 0.111168 |
| }, |
| "model.layers.28.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999797, |
| "clip_improvement": 0.25096 |
| }, |
| "model.layers.28.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999837, |
| "clip_improvement": 0.250603 |
| }, |
| "model.layers.28.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.99985, |
| "clip_improvement": 0.250364 |
| }, |
| "model.layers.28.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954224, |
| "clip_improvement": 0.112371 |
| }, |
| "model.layers.28.self_attn.o_proj": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4718592, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999977, |
| "clip_improvement": 0.353555 |
| }, |
| "model.layers.28.self_attn.q_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6815744, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.99987, |
| "clip_improvement": 0.251118 |
| }, |
| "model.layers.28.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954681, |
| "clip_improvement": 0.110762 |
| }, |
| "model.layers.29.mlp.down_proj": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 16515072, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999867, |
| "clip_improvement": 0.354033 |
| }, |
| "model.layers.29.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999813, |
| "clip_improvement": 0.250075 |
| }, |
| "model.layers.29.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999828, |
| "clip_improvement": 0.250467 |
| }, |
| "model.layers.29.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954102, |
| "clip_improvement": 0.112702 |
| }, |
| "model.layers.29.self_attn.o_proj": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4718592, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 1.0, |
| "clip_improvement": 0.353919 |
| }, |
| "model.layers.29.self_attn.q_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6815744, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999855, |
| "clip_improvement": 0.252049 |
| }, |
| "model.layers.29.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955322, |
| "clip_improvement": 0.110842 |
| }, |
| "model.layers.3.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999823, |
| "clip_improvement": 0.250621 |
| }, |
| "model.layers.3.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999808, |
| "clip_improvement": 0.250524 |
| }, |
| "model.layers.3.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999839, |
| "clip_improvement": 0.250106 |
| }, |
| "model.layers.3.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.956268, |
| "clip_improvement": 0.111218 |
| }, |
| "model.layers.3.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953636, |
| "clip_improvement": 0.111545 |
| }, |
| "model.layers.3.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955292, |
| "clip_improvement": 0.111747 |
| }, |
| "model.layers.3.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953796, |
| "clip_improvement": 0.111013 |
| }, |
| "model.layers.30.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999786, |
| "clip_improvement": 0.252717 |
| }, |
| "model.layers.30.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999823, |
| "clip_improvement": 0.250554 |
| }, |
| "model.layers.30.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.99983, |
| "clip_improvement": 0.250294 |
| }, |
| "model.layers.30.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955566, |
| "clip_improvement": 0.112461 |
| }, |
| "model.layers.30.self_attn.o_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6815744, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999802, |
| "clip_improvement": 0.250114 |
| }, |
| "model.layers.30.self_attn.q_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6815744, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999825, |
| "clip_improvement": 0.251371 |
| }, |
| "model.layers.30.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.959351, |
| "clip_improvement": 0.111455 |
| }, |
| "model.layers.31.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999821, |
| "clip_improvement": 0.252817 |
| }, |
| "model.layers.31.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999854, |
| "clip_improvement": 0.250397 |
| }, |
| "model.layers.31.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999878, |
| "clip_improvement": 0.250394 |
| }, |
| "model.layers.31.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954468, |
| "clip_improvement": 0.111969 |
| }, |
| "model.layers.31.self_attn.o_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6815744, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999825, |
| "clip_improvement": 0.252438 |
| }, |
| "model.layers.31.self_attn.q_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 6815744, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999802, |
| "clip_improvement": 0.251303 |
| }, |
| "model.layers.31.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.957611, |
| "clip_improvement": 0.112046 |
| }, |
| "model.layers.4.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999821, |
| "clip_improvement": 0.251359 |
| }, |
| "model.layers.4.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999797, |
| "clip_improvement": 0.251122 |
| }, |
| "model.layers.4.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999852, |
| "clip_improvement": 0.250753 |
| }, |
| "model.layers.4.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95459, |
| "clip_improvement": 0.112584 |
| }, |
| "model.layers.4.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953201, |
| "clip_improvement": 0.111462 |
| }, |
| "model.layers.4.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954849, |
| "clip_improvement": 0.111688 |
| }, |
| "model.layers.4.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953674, |
| "clip_improvement": 0.111637 |
| }, |
| "model.layers.5.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999747, |
| "clip_improvement": 0.250899 |
| }, |
| "model.layers.5.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999747, |
| "clip_improvement": 0.251304 |
| }, |
| "model.layers.5.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999813, |
| "clip_improvement": 0.250499 |
| }, |
| "model.layers.5.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.956085, |
| "clip_improvement": 0.1119 |
| }, |
| "model.layers.5.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954559, |
| "clip_improvement": 0.111708 |
| }, |
| "model.layers.5.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955078, |
| "clip_improvement": 0.111334 |
| }, |
| "model.layers.5.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953003, |
| "clip_improvement": 0.111743 |
| }, |
| "model.layers.6.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999797, |
| "clip_improvement": 0.251373 |
| }, |
| "model.layers.6.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999808, |
| "clip_improvement": 0.250786 |
| }, |
| "model.layers.6.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999815, |
| "clip_improvement": 0.250891 |
| }, |
| "model.layers.6.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955719, |
| "clip_improvement": 0.11243 |
| }, |
| "model.layers.6.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953156, |
| "clip_improvement": 0.11102 |
| }, |
| "model.layers.6.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955269, |
| "clip_improvement": 0.112038 |
| }, |
| "model.layers.6.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955841, |
| "clip_improvement": 0.111308 |
| }, |
| "model.layers.7.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999802, |
| "clip_improvement": 0.251257 |
| }, |
| "model.layers.7.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999819, |
| "clip_improvement": 0.250713 |
| }, |
| "model.layers.7.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999843, |
| "clip_improvement": 0.250846 |
| }, |
| "model.layers.7.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954742, |
| "clip_improvement": 0.111861 |
| }, |
| "model.layers.7.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954109, |
| "clip_improvement": 0.111923 |
| }, |
| "model.layers.7.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954811, |
| "clip_improvement": 0.111631 |
| }, |
| "model.layers.7.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955536, |
| "clip_improvement": 0.11122 |
| }, |
| "model.layers.8.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999786, |
| "clip_improvement": 0.250944 |
| }, |
| "model.layers.8.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954633, |
| "clip_improvement": 0.111662 |
| }, |
| "model.layers.8.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.99983, |
| "clip_improvement": 0.250718 |
| }, |
| "model.layers.8.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953888, |
| "clip_improvement": 0.111444 |
| }, |
| "model.layers.8.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954124, |
| "clip_improvement": 0.111506 |
| }, |
| "model.layers.8.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954231, |
| "clip_improvement": 0.111554 |
| }, |
| "model.layers.8.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953949, |
| "clip_improvement": 0.111854 |
| }, |
| "model.layers.9.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999869, |
| "clip_improvement": 0.250692 |
| }, |
| "model.layers.9.mlp.gate_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999804, |
| "clip_improvement": 0.250961 |
| }, |
| "model.layers.9.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999865, |
| "clip_improvement": 0.250806 |
| }, |
| "model.layers.9.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953949, |
| "clip_improvement": 0.111844 |
| }, |
| "model.layers.9.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95401, |
| "clip_improvement": 0.111623 |
| }, |
| "model.layers.9.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954811, |
| "clip_improvement": 0.111774 |
| }, |
| "model.layers.9.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954956, |
| "clip_improvement": 0.110794 |
| } |
| } |
| } |
