| { |
| "schema": "dynquant_checkpoint_v2", |
| "dynquant_core": "0.4.0", |
| "quantization_config": { |
| "quant_method": "dynquant", |
| "checkpoint_format": "dynquant-packed", |
| "schema_version": 1, |
| "version": "0.4.0", |
| "group_size": 128, |
| "symmetric": false, |
| "lm_head_quantized": true, |
| "modules_to_not_convert": [], |
| "modules": { |
| "lm_head": { |
| "bits": 8, |
| "out_features": 32768 |
| }, |
| "model.embed_tokens": { |
| "bits": 4, |
| "out_features": 32768 |
| }, |
| "model.layers.0.mlp.down_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.0.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.0.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.0.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.0.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.0.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.0.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.1.mlp.down_proj": { |
| "bits": 8, |
| "out_features": 4096 |
| }, |
| "model.layers.1.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.1.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.1.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.1.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.1.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.1.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.10.mlp.down_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.10.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.10.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.10.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.10.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.10.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.10.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.11.mlp.down_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.11.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.11.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.11.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.11.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.11.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.11.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.12.mlp.down_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.12.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.12.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.12.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.12.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.12.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.12.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.13.mlp.down_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.13.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.13.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.13.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.13.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.13.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.13.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.14.mlp.down_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.14.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.14.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.14.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.14.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.14.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.14.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.15.mlp.down_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.15.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.15.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.15.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.15.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.15.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.15.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.16.mlp.down_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.16.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.16.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.16.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.16.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.16.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.16.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.17.mlp.down_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.17.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.17.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.17.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.17.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.17.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.17.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.18.mlp.down_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.18.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.18.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.18.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.18.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.18.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.18.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.19.mlp.down_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.19.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.19.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.19.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.19.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.19.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.19.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.2.mlp.down_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.2.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.2.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.2.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.2.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.2.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.2.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.20.mlp.down_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.20.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.20.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.20.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.20.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.20.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.20.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.21.mlp.down_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.21.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.21.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.21.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.21.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.21.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.21.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.22.mlp.down_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.22.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.22.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.22.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.22.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.22.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.22.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.23.mlp.down_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.23.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.23.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.23.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.23.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.23.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.23.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.24.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.24.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.24.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.24.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.24.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.24.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.24.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.25.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.25.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.25.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.25.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.25.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.25.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.25.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.26.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.26.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.26.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.26.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.26.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.26.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.26.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.27.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.27.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.27.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.27.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.27.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.27.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.27.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.28.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.28.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.28.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.28.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.28.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.28.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.28.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.29.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.29.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.29.mlp.up_proj": { |
| "bits": 3, |
| "out_features": 14336 |
| }, |
| "model.layers.29.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.29.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.29.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.29.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.3.mlp.down_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.3.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.3.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.3.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.3.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.3.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.3.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.30.mlp.down_proj": { |
| "bits": 3, |
| "out_features": 4096 |
| }, |
| "model.layers.30.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.30.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.30.self_attn.k_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.30.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.30.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.30.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.31.mlp.down_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.31.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.31.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.31.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.31.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.31.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.31.self_attn.v_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.4.mlp.down_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.4.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.4.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.4.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.4.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.4.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.4.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.5.mlp.down_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.5.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.5.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.5.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.5.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.5.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.5.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.6.mlp.down_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.6.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.6.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.6.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.6.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.6.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.6.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.7.mlp.down_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.7.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.7.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.7.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.7.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.7.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.7.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.8.mlp.down_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.8.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.8.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.8.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.8.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.8.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.8.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.9.mlp.down_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.9.mlp.gate_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.9.mlp.up_proj": { |
| "bits": 4, |
| "out_features": 14336 |
| }, |
| "model.layers.9.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.9.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.9.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 4096 |
| }, |
| "model.layers.9.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| } |
| } |
| }, |
| "files": [ |
| "model.safetensors" |
| ], |
| "accounting": { |
| "packed_nbytes": 3963617280, |
| "dense_nbytes": 532480, |
| "total_nbytes": 3964149760, |
| "quantized_elements": 7247757312, |
| "average_bits": 4.375 |
| }, |
| "tied": {}, |
| "expert_banks": [], |
| "provenance": { |
| "dynquant_core": "0.4.0", |
| "model": "/workspace/runs/s4/mistral7b-v03.text2sql/merged", |
| "stats": null, |
| "allocator": "sensitivity", |
| "widths_from": "/workspace/runs/s4/panel-mistral/maps/dq_4b.json [3964674048]", |
| "predicted_nbytes": 3964149760 |
| }, |
| "layers": { |
| "lm_head": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 32768, |
| 4096 |
| ], |
| "num_rows": 32768, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 138412032, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.embed_tokens": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 32768, |
| 4096 |
| ], |
| "num_rows": 32768, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 71303168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.9277, |
| "clip_improvement": 0.111654 |
| }, |
| "model.layers.0.mlp.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954995, |
| "clip_improvement": 0.111722 |
| }, |
| "model.layers.0.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955314, |
| "clip_improvement": 0.111629 |
| }, |
| "model.layers.0.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999852, |
| "clip_improvement": 0.250389 |
| }, |
| "model.layers.0.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.89209, |
| "clip_improvement": 0.21042 |
| }, |
| "model.layers.0.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955269, |
| "clip_improvement": 0.111095 |
| }, |
| "model.layers.0.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.870163, |
| "clip_improvement": 0.248147 |
| }, |
| "model.layers.0.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.1.mlp.down_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 60555264, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.1.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95469, |
| "clip_improvement": 0.111778 |
| }, |
| "model.layers.1.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.99976, |
| "clip_improvement": 0.250698 |
| }, |
| "model.layers.1.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.1.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954872, |
| "clip_improvement": 0.111292 |
| }, |
| "model.layers.1.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.926826, |
| "clip_improvement": 0.134538 |
| }, |
| "model.layers.1.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.10.mlp.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955401, |
| "clip_improvement": 0.111817 |
| }, |
| "model.layers.10.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95517, |
| "clip_improvement": 0.111777 |
| }, |
| "model.layers.10.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954806, |
| "clip_improvement": 0.111762 |
| }, |
| "model.layers.10.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.10.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954659, |
| "clip_improvement": 0.111894 |
| }, |
| "model.layers.10.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955727, |
| "clip_improvement": 0.111726 |
| }, |
| "model.layers.10.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.11.mlp.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955065, |
| "clip_improvement": 0.111732 |
| }, |
| "model.layers.11.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955216, |
| "clip_improvement": 0.111866 |
| }, |
| "model.layers.11.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955654, |
| "clip_improvement": 0.111685 |
| }, |
| "model.layers.11.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.11.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954651, |
| "clip_improvement": 0.111641 |
| }, |
| "model.layers.11.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.9543, |
| "clip_improvement": 0.11162 |
| }, |
| "model.layers.11.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.12.mlp.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954756, |
| "clip_improvement": 0.111839 |
| }, |
| "model.layers.12.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954989, |
| "clip_improvement": 0.111947 |
| }, |
| "model.layers.12.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955111, |
| "clip_improvement": 0.111493 |
| }, |
| "model.layers.12.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.12.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954231, |
| "clip_improvement": 0.111711 |
| }, |
| "model.layers.12.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954811, |
| "clip_improvement": 0.110958 |
| }, |
| "model.layers.12.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.13.mlp.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955226, |
| "clip_improvement": 0.111729 |
| }, |
| "model.layers.13.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955662, |
| "clip_improvement": 0.111942 |
| }, |
| "model.layers.13.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95534, |
| "clip_improvement": 0.111718 |
| }, |
| "model.layers.13.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.13.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954491, |
| "clip_improvement": 0.11161 |
| }, |
| "model.layers.13.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955147, |
| "clip_improvement": 0.111867 |
| }, |
| "model.layers.13.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.14.mlp.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954919, |
| "clip_improvement": 0.111965 |
| }, |
| "model.layers.14.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955198, |
| "clip_improvement": 0.11176 |
| }, |
| "model.layers.14.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955808, |
| "clip_improvement": 0.111607 |
| }, |
| "model.layers.14.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.14.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954231, |
| "clip_improvement": 0.111575 |
| }, |
| "model.layers.14.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95491, |
| "clip_improvement": 0.111332 |
| }, |
| "model.layers.14.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.15.mlp.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95513, |
| "clip_improvement": 0.111781 |
| }, |
| "model.layers.15.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955144, |
| "clip_improvement": 0.112064 |
| }, |
| "model.layers.15.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954671, |
| "clip_improvement": 0.111558 |
| }, |
| "model.layers.15.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.15.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954422, |
| "clip_improvement": 0.11157 |
| }, |
| "model.layers.15.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954803, |
| "clip_improvement": 0.111575 |
| }, |
| "model.layers.15.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.16.mlp.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954777, |
| "clip_improvement": 0.112005 |
| }, |
| "model.layers.16.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955168, |
| "clip_improvement": 0.11189 |
| }, |
| "model.layers.16.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954926, |
| "clip_improvement": 0.111514 |
| }, |
| "model.layers.16.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.16.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954048, |
| "clip_improvement": 0.111673 |
| }, |
| "model.layers.16.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954651, |
| "clip_improvement": 0.111957 |
| }, |
| "model.layers.16.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.17.mlp.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955083, |
| "clip_improvement": 0.111797 |
| }, |
| "model.layers.17.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954817, |
| "clip_improvement": 0.111652 |
| }, |
| "model.layers.17.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954939, |
| "clip_improvement": 0.11166 |
| }, |
| "model.layers.17.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.17.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95369, |
| "clip_improvement": 0.111864 |
| }, |
| "model.layers.17.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954697, |
| "clip_improvement": 0.111357 |
| }, |
| "model.layers.17.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.18.mlp.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95503, |
| "clip_improvement": 0.111924 |
| }, |
| "model.layers.18.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954821, |
| "clip_improvement": 0.11173 |
| }, |
| "model.layers.18.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955067, |
| "clip_improvement": 0.111609 |
| }, |
| "model.layers.18.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.18.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953995, |
| "clip_improvement": 0.111652 |
| }, |
| "model.layers.18.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955261, |
| "clip_improvement": 0.111183 |
| }, |
| "model.layers.18.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.19.mlp.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954422, |
| "clip_improvement": 0.111709 |
| }, |
| "model.layers.19.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954505, |
| "clip_improvement": 0.111557 |
| }, |
| "model.layers.19.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954378, |
| "clip_improvement": 0.111655 |
| }, |
| "model.layers.19.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.19.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95388, |
| "clip_improvement": 0.112022 |
| }, |
| "model.layers.19.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953545, |
| "clip_improvement": 0.111476 |
| }, |
| "model.layers.19.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.2.mlp.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954793, |
| "clip_improvement": 0.111832 |
| }, |
| "model.layers.2.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954213, |
| "clip_improvement": 0.11166 |
| }, |
| "model.layers.2.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954668, |
| "clip_improvement": 0.111505 |
| }, |
| "model.layers.2.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.2.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954819, |
| "clip_improvement": 0.111629 |
| }, |
| "model.layers.2.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955818, |
| "clip_improvement": 0.111097 |
| }, |
| "model.layers.2.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.20.mlp.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954335, |
| "clip_improvement": 0.111929 |
| }, |
| "model.layers.20.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954113, |
| "clip_improvement": 0.111808 |
| }, |
| "model.layers.20.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954649, |
| "clip_improvement": 0.111742 |
| }, |
| "model.layers.20.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.20.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954109, |
| "clip_improvement": 0.111827 |
| }, |
| "model.layers.20.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95533, |
| "clip_improvement": 0.112181 |
| }, |
| "model.layers.20.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.21.mlp.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954283, |
| "clip_improvement": 0.111674 |
| }, |
| "model.layers.21.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954363, |
| "clip_improvement": 0.11175 |
| }, |
| "model.layers.21.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954025, |
| "clip_improvement": 0.111682 |
| }, |
| "model.layers.21.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.21.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955643, |
| "clip_improvement": 0.112023 |
| }, |
| "model.layers.21.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955002, |
| "clip_improvement": 0.111644 |
| }, |
| "model.layers.21.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.22.mlp.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954025, |
| "clip_improvement": 0.11182 |
| }, |
| "model.layers.22.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954317, |
| "clip_improvement": 0.111939 |
| }, |
| "model.layers.22.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954435, |
| "clip_improvement": 0.111821 |
| }, |
| "model.layers.22.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.22.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954323, |
| "clip_improvement": 0.111696 |
| }, |
| "model.layers.22.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954758, |
| "clip_improvement": 0.111945 |
| }, |
| "model.layers.22.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.23.mlp.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953953, |
| "clip_improvement": 0.11189 |
| }, |
| "model.layers.23.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954529, |
| "clip_improvement": 0.111942 |
| }, |
| "model.layers.23.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953842, |
| "clip_improvement": 0.111662 |
| }, |
| "model.layers.23.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.23.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953354, |
| "clip_improvement": 0.111375 |
| }, |
| "model.layers.23.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955055, |
| "clip_improvement": 0.111608 |
| }, |
| "model.layers.23.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.24.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999841, |
| "clip_improvement": 0.250641 |
| }, |
| "model.layers.24.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954001, |
| "clip_improvement": 0.11202 |
| }, |
| "model.layers.24.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95413, |
| "clip_improvement": 0.111661 |
| }, |
| "model.layers.24.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952911, |
| "clip_improvement": 0.112427 |
| }, |
| "model.layers.24.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954483, |
| "clip_improvement": 0.111913 |
| }, |
| "model.layers.24.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953903, |
| "clip_improvement": 0.111511 |
| }, |
| "model.layers.24.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.25.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999882, |
| "clip_improvement": 0.250323 |
| }, |
| "model.layers.25.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954677, |
| "clip_improvement": 0.111964 |
| }, |
| "model.layers.25.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999806, |
| "clip_improvement": 0.250062 |
| }, |
| "model.layers.25.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955902, |
| "clip_improvement": 0.112661 |
| }, |
| "model.layers.25.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952873, |
| "clip_improvement": 0.111868 |
| }, |
| "model.layers.25.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953674, |
| "clip_improvement": 0.11193 |
| }, |
| "model.layers.25.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.26.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999823, |
| "clip_improvement": 0.250825 |
| }, |
| "model.layers.26.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95459, |
| "clip_improvement": 0.111877 |
| }, |
| "model.layers.26.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.99985, |
| "clip_improvement": 0.250106 |
| }, |
| "model.layers.26.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.26.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95459, |
| "clip_improvement": 0.112125 |
| }, |
| "model.layers.26.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954407, |
| "clip_improvement": 0.111909 |
| }, |
| "model.layers.26.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.27.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999837, |
| "clip_improvement": 0.250383 |
| }, |
| "model.layers.27.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954333, |
| "clip_improvement": 0.111933 |
| }, |
| "model.layers.27.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999876, |
| "clip_improvement": 0.250137 |
| }, |
| "model.layers.27.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95462, |
| "clip_improvement": 0.11258 |
| }, |
| "model.layers.27.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953629, |
| "clip_improvement": 0.111731 |
| }, |
| "model.layers.27.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954422, |
| "clip_improvement": 0.111798 |
| }, |
| "model.layers.27.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.28.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999797, |
| "clip_improvement": 0.25096 |
| }, |
| "model.layers.28.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954136, |
| "clip_improvement": 0.111981 |
| }, |
| "model.layers.28.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.99985, |
| "clip_improvement": 0.250364 |
| }, |
| "model.layers.28.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.28.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952583, |
| "clip_improvement": 0.11159 |
| }, |
| "model.layers.28.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954071, |
| "clip_improvement": 0.111935 |
| }, |
| "model.layers.28.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.29.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999806, |
| "clip_improvement": 0.250502 |
| }, |
| "model.layers.29.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954518, |
| "clip_improvement": 0.111792 |
| }, |
| "model.layers.29.mlp.up_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999828, |
| "clip_improvement": 0.250467 |
| }, |
| "model.layers.29.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954102, |
| "clip_improvement": 0.112702 |
| }, |
| "model.layers.29.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953529, |
| "clip_improvement": 0.111742 |
| }, |
| "model.layers.29.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955261, |
| "clip_improvement": 0.111977 |
| }, |
| "model.layers.29.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.3.mlp.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954235, |
| "clip_improvement": 0.111783 |
| }, |
| "model.layers.3.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95386, |
| "clip_improvement": 0.111778 |
| }, |
| "model.layers.3.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954762, |
| "clip_improvement": 0.111763 |
| }, |
| "model.layers.3.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.3.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953636, |
| "clip_improvement": 0.111545 |
| }, |
| "model.layers.3.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955292, |
| "clip_improvement": 0.111747 |
| }, |
| "model.layers.3.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.30.mlp.down_proj": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 23855104, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999786, |
| "clip_improvement": 0.252717 |
| }, |
| "model.layers.30.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954557, |
| "clip_improvement": 0.111757 |
| }, |
| "model.layers.30.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954429, |
| "clip_improvement": 0.11181 |
| }, |
| "model.layers.30.self_attn.k_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955566, |
| "clip_improvement": 0.112461 |
| }, |
| "model.layers.30.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954994, |
| "clip_improvement": 0.111794 |
| }, |
| "model.layers.30.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953728, |
| "clip_improvement": 0.111865 |
| }, |
| "model.layers.30.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.31.mlp.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955824, |
| "clip_improvement": 0.110921 |
| }, |
| "model.layers.31.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954705, |
| "clip_improvement": 0.111719 |
| }, |
| "model.layers.31.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954795, |
| "clip_improvement": 0.111779 |
| }, |
| "model.layers.31.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.31.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95488, |
| "clip_improvement": 0.112445 |
| }, |
| "model.layers.31.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954491, |
| "clip_improvement": 0.112218 |
| }, |
| "model.layers.31.self_attn.v_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 2228224, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.957611, |
| "clip_improvement": 0.112046 |
| }, |
| "model.layers.4.mlp.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954215, |
| "clip_improvement": 0.111951 |
| }, |
| "model.layers.4.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954629, |
| "clip_improvement": 0.112151 |
| }, |
| "model.layers.4.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954614, |
| "clip_improvement": 0.111802 |
| }, |
| "model.layers.4.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.4.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953201, |
| "clip_improvement": 0.111462 |
| }, |
| "model.layers.4.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954849, |
| "clip_improvement": 0.111688 |
| }, |
| "model.layers.4.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.5.mlp.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953953, |
| "clip_improvement": 0.111808 |
| }, |
| "model.layers.5.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954644, |
| "clip_improvement": 0.111899 |
| }, |
| "model.layers.5.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955399, |
| "clip_improvement": 0.111783 |
| }, |
| "model.layers.5.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.5.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954559, |
| "clip_improvement": 0.111708 |
| }, |
| "model.layers.5.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955078, |
| "clip_improvement": 0.111334 |
| }, |
| "model.layers.5.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.6.mlp.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954808, |
| "clip_improvement": 0.111979 |
| }, |
| "model.layers.6.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953565, |
| "clip_improvement": 0.111873 |
| }, |
| "model.layers.6.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954544, |
| "clip_improvement": 0.111683 |
| }, |
| "model.layers.6.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.6.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953156, |
| "clip_improvement": 0.11102 |
| }, |
| "model.layers.6.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955269, |
| "clip_improvement": 0.112038 |
| }, |
| "model.layers.6.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.7.mlp.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954546, |
| "clip_improvement": 0.111811 |
| }, |
| "model.layers.7.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954834, |
| "clip_improvement": 0.111859 |
| }, |
| "model.layers.7.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955078, |
| "clip_improvement": 0.111695 |
| }, |
| "model.layers.7.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.7.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954109, |
| "clip_improvement": 0.111923 |
| }, |
| "model.layers.7.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954811, |
| "clip_improvement": 0.111631 |
| }, |
| "model.layers.7.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.8.mlp.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954618, |
| "clip_improvement": 0.111939 |
| }, |
| "model.layers.8.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954633, |
| "clip_improvement": 0.111662 |
| }, |
| "model.layers.8.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954692, |
| "clip_improvement": 0.111688 |
| }, |
| "model.layers.8.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.8.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954124, |
| "clip_improvement": 0.111506 |
| }, |
| "model.layers.8.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954231, |
| "clip_improvement": 0.111554 |
| }, |
| "model.layers.8.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.9.mlp.down_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 14336, |
| "logical_shape": [ |
| 4096, |
| 14336 |
| ], |
| "num_rows": 4096, |
| "num_groups": 112, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954695, |
| "clip_improvement": 0.112018 |
| }, |
| "model.layers.9.mlp.gate_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954684, |
| "clip_improvement": 0.111766 |
| }, |
| "model.layers.9.mlp.up_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 14336, |
| 4096 |
| ], |
| "num_rows": 14336, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 31195136, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954535, |
| "clip_improvement": 0.111764 |
| }, |
| "model.layers.9.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.9.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95401, |
| "clip_improvement": 0.111623 |
| }, |
| "model.layers.9.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 4096, |
| 4096 |
| ], |
| "num_rows": 4096, |
| "num_groups": 32, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 8912896, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954811, |
| "clip_improvement": 0.111774 |
| }, |
| "model.layers.9.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 4096, |
| "logical_shape": [ |
| 1024, |
| 4096 |
| ], |
| "num_rows": 1024, |
| "num_groups": 32, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 4325376, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| } |
| } |
| } |
