{ "schema": "dynquant_checkpoint_v2", "dynquant_core": "0.5.2", "quantization_config": { "quant_method": "dynquant", "checkpoint_format": "dynquant-packed", "schema_version": 1, "version": "0.5.2", "group_size": 128, "symmetric": false, "lm_head_quantized": true, "modules_to_not_convert": [], "modules": { "lm_head": { "bits": 8, "out_features": 248320 }, "model.embed_tokens": { "bits": 4, "out_features": 248320 }, "model.layers.0.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.0.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.0.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.0.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.0.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.0.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.0.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.0.mlp.up_proj": { "bits": 2, "out_features": 17408 }, "model.layers.1.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.1.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.1.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.1.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.1.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.1.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.1.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.1.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.10.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.10.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.10.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.10.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.10.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.10.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.10.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.10.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.11.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.11.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.11.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.11.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.11.self_attn.o_proj": { "bits": 4, "out_features": 5120 }, "model.layers.11.self_attn.q_proj": { "bits": 4, "out_features": 12288 }, "model.layers.11.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.12.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.12.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.12.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.12.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.12.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.12.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.12.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.12.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.13.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.13.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.13.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.13.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.13.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.13.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.13.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.13.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.14.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.14.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.14.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.14.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.14.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.14.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.14.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.14.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.15.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.15.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.15.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.15.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.15.self_attn.o_proj": { "bits": 4, "out_features": 5120 }, "model.layers.15.self_attn.q_proj": { "bits": 4, "out_features": 12288 }, "model.layers.15.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.16.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.16.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.16.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.16.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.16.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.16.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.16.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.16.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.17.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.17.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.17.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.17.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.17.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.17.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.17.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.17.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.18.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.18.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.18.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.18.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.18.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.18.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.18.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.18.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.19.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.19.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.19.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.19.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.19.self_attn.o_proj": { "bits": 4, "out_features": 5120 }, "model.layers.19.self_attn.q_proj": { "bits": 4, "out_features": 12288 }, "model.layers.19.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.2.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.2.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.2.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.2.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.2.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.2.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.2.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.2.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.20.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.20.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.20.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.20.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.20.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.20.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.20.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.20.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.21.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.21.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.21.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.21.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.21.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.21.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.21.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.21.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.22.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.22.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.22.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.22.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.22.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.22.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.22.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.22.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.23.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.23.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.23.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.23.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.23.self_attn.o_proj": { "bits": 4, "out_features": 5120 }, "model.layers.23.self_attn.q_proj": { "bits": 4, "out_features": 12288 }, "model.layers.23.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.24.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.24.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.24.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.24.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.24.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.24.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.24.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.24.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.25.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.25.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.25.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.25.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.25.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.25.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.25.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.25.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.26.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.26.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.26.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.26.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.26.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.26.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.26.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.26.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.27.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.27.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.27.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.27.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.27.self_attn.o_proj": { "bits": 4, "out_features": 5120 }, "model.layers.27.self_attn.q_proj": { "bits": 4, "out_features": 12288 }, "model.layers.27.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.28.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.28.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.28.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.28.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.28.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.28.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.28.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.28.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.29.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.29.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.29.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.29.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.29.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.29.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.29.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.29.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.3.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.3.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.3.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.3.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.3.self_attn.o_proj": { "bits": 4, "out_features": 5120 }, "model.layers.3.self_attn.q_proj": { "bits": 4, "out_features": 12288 }, "model.layers.3.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.30.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.30.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.30.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.30.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.30.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.30.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.30.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.30.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.31.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.31.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.31.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.31.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.31.self_attn.o_proj": { "bits": 4, "out_features": 5120 }, "model.layers.31.self_attn.q_proj": { "bits": 4, "out_features": 12288 }, "model.layers.31.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.32.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.32.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.32.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.32.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.32.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.32.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.32.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.32.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.33.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.33.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.33.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.33.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.33.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.33.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.33.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.33.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.34.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.34.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.34.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.34.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.34.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.34.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.34.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.34.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.35.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.35.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.35.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.35.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.35.self_attn.o_proj": { "bits": 4, "out_features": 5120 }, "model.layers.35.self_attn.q_proj": { "bits": 4, "out_features": 12288 }, "model.layers.35.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.36.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.36.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.36.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.36.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.36.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.36.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.36.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.36.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.37.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.37.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.37.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.37.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.37.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.37.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.37.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.37.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.38.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.38.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.38.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.38.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.38.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.38.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.38.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.38.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.39.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.39.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.39.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.39.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.39.self_attn.o_proj": { "bits": 4, "out_features": 5120 }, "model.layers.39.self_attn.q_proj": { "bits": 4, "out_features": 12288 }, "model.layers.39.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.4.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.4.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.4.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.4.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.4.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.4.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.4.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.4.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.40.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.40.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.40.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.40.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.40.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.40.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.40.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.40.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.41.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.41.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.41.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.41.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.41.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.41.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.41.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.41.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.42.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.42.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.42.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.42.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.42.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.42.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.42.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.42.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.43.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.43.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.43.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.43.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.43.self_attn.o_proj": { "bits": 4, "out_features": 5120 }, "model.layers.43.self_attn.q_proj": { "bits": 4, "out_features": 12288 }, "model.layers.43.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.44.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.44.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.44.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.44.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.44.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.44.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.44.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.44.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.45.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.45.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.45.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.45.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.45.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.45.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.45.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.45.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.46.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.46.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.46.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.46.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.46.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.46.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.46.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.46.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.47.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.47.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.47.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.47.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.47.self_attn.o_proj": { "bits": 4, "out_features": 5120 }, "model.layers.47.self_attn.q_proj": { "bits": 4, "out_features": 12288 }, "model.layers.47.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.48.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.48.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.48.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.48.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.48.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.48.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.48.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.48.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.49.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.49.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.49.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.49.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.49.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.49.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.49.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.49.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.5.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.5.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.5.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.5.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.5.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.5.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.5.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.5.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.50.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.50.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.50.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.50.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.50.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.50.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.50.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.50.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.51.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.51.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.51.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.51.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.51.self_attn.o_proj": { "bits": 4, "out_features": 5120 }, "model.layers.51.self_attn.q_proj": { "bits": 4, "out_features": 12288 }, "model.layers.51.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.52.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.52.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.52.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.52.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.52.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.52.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.52.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.52.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.53.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.53.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.53.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.53.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.53.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.53.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.53.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.53.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.54.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.54.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.54.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.54.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.54.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.54.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.54.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.54.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.55.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.55.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.55.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.55.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.55.self_attn.o_proj": { "bits": 3, "out_features": 5120 }, "model.layers.55.self_attn.q_proj": { "bits": 3, "out_features": 12288 }, "model.layers.55.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.56.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.56.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.56.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.56.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.56.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.56.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.56.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.56.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.57.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.57.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.57.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.57.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.57.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.57.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.57.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.57.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.58.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.58.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.58.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.58.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.58.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.58.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.58.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.58.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.59.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.59.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.59.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.59.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.59.self_attn.o_proj": { "bits": 4, "out_features": 5120 }, "model.layers.59.self_attn.q_proj": { "bits": 4, "out_features": 12288 }, "model.layers.59.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.6.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.6.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.6.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.6.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.6.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.6.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.6.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.6.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.60.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.60.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.60.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.60.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.60.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.60.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.60.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.60.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.61.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.61.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.61.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.61.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.61.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.61.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.61.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.61.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.62.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.62.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.62.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.62.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.62.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.62.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.62.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.62.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.63.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.63.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.63.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.63.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.63.self_attn.o_proj": { "bits": 4, "out_features": 5120 }, "model.layers.63.self_attn.q_proj": { "bits": 4, "out_features": 12288 }, "model.layers.63.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.7.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.7.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.7.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.7.self_attn.k_proj": { "bits": 4, "out_features": 1024 }, "model.layers.7.self_attn.o_proj": { "bits": 4, "out_features": 5120 }, "model.layers.7.self_attn.q_proj": { "bits": 4, "out_features": 12288 }, "model.layers.7.self_attn.v_proj": { "bits": 4, "out_features": 1024 }, "model.layers.8.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.8.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.8.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.8.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.8.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.8.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.8.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.8.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.9.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.9.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.9.linear_attn.in_proj_qkv": { "bits": 4, "out_features": 10240 }, "model.layers.9.linear_attn.in_proj_z": { "bits": 4, "out_features": 6144 }, "model.layers.9.linear_attn.out_proj": { "bits": 4, "out_features": 5120 }, "model.layers.9.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.9.mlp.gate_proj": { "bits": 4, "out_features": 17408 }, "model.layers.9.mlp.up_proj": { "bits": 3, "out_features": 17408 } } }, "files": [ "model-00001-of-00004.safetensors", "model-00002-of-00004.safetensors", "model-00003-of-00004.safetensors", "model-00004-of-00004.safetensors", "model.safetensors.index.json" ], "accounting": { "packed_nbytes": 13442334720, "dense_nbytes": 5291008, "total_nbytes": 13447625728, "quantized_elements": 26893352960, "average_bits": 3.998708 }, "tied": {}, "expert_banks": [], "provenance": { "dynquant_core": "0.5.2", "model": "/workspace/runs-s2/qwen38-27b.text2sql/merged", "stats": "/workspace/runs-s2/qwen38-27b.text2sql/stats/dynquant_stats.json", "allocator": "rank_product", "widths_from": "4.00 avg bits", "predicted_nbytes": 13447625728 }, "layers": { "lm_head": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 248320, 5120 ], "num_rows": 248320, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1311129600, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.embed_tokens": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 248320, 5120 ], "num_rows": 248320, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 675430400, "bits_per_weight": 4.25, "clipped_fraction": 0.953984, "clip_improvement": 0.112084 }, "model.layers.0.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.0.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.0.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.95415, "clip_improvement": 0.111904 }, "model.layers.0.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954659, "clip_improvement": 0.111781 }, "model.layers.0.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.956059, "clip_improvement": 0.110666 }, "model.layers.0.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999849, "clip_improvement": 0.250686 }, "model.layers.0.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.250003 }, "model.layers.0.mlp.up_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999978, "clip_improvement": 0.353266 }, "model.layers.1.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.1.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.1.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.954595, "clip_improvement": 0.112052 }, "model.layers.1.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954106, "clip_improvement": 0.111822 }, "model.layers.1.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954895, "clip_improvement": 0.111771 }, "model.layers.1.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999822, "clip_improvement": 0.251641 }, "model.layers.1.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.953819, "clip_improvement": 0.111691 }, "model.layers.1.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999848, "clip_improvement": 0.249995 }, "model.layers.10.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.10.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.10.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955293, "clip_improvement": 0.111847 }, "model.layers.10.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.95544, "clip_improvement": 0.111705 }, "model.layers.10.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955514, "clip_improvement": 0.111696 }, "model.layers.10.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999858, "clip_improvement": 0.250209 }, "model.layers.10.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.95357, "clip_improvement": 0.111593 }, "model.layers.10.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999848, "clip_improvement": 0.250044 }, "model.layers.11.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999842, "clip_improvement": 0.250375 }, "model.layers.11.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954025, "clip_improvement": 0.111565 }, "model.layers.11.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999866, "clip_improvement": 0.24994 }, "model.layers.11.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.956763, "clip_improvement": 0.111992 }, "model.layers.11.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954488, "clip_improvement": 0.111156 }, "model.layers.11.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 33423360, "bits_per_weight": 4.25, "clipped_fraction": 0.955436, "clip_improvement": 0.111732 }, "model.layers.11.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.95564, "clip_improvement": 0.111539 }, "model.layers.12.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.12.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.12.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.954878, "clip_improvement": 0.111612 }, "model.layers.12.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955335, "clip_improvement": 0.111938 }, "model.layers.12.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954671, "clip_improvement": 0.111786 }, "model.layers.12.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999842, "clip_improvement": 0.250405 }, "model.layers.12.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.953875, "clip_improvement": 0.111618 }, "model.layers.12.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999832, "clip_improvement": 0.249959 }, "model.layers.13.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.13.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.13.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955083, "clip_improvement": 0.111819 }, "model.layers.13.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954443, "clip_improvement": 0.111811 }, "model.layers.13.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955811, "clip_improvement": 0.111977 }, "model.layers.13.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999806, "clip_improvement": 0.249997 }, "model.layers.13.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954267, "clip_improvement": 0.111558 }, "model.layers.13.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999848, "clip_improvement": 0.25019 }, "model.layers.14.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.14.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.14.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955312, "clip_improvement": 0.111958 }, "model.layers.14.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954594, "clip_improvement": 0.112234 }, "model.layers.14.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955127, "clip_improvement": 0.111368 }, "model.layers.14.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999855, "clip_improvement": 0.250201 }, "model.layers.14.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.95388, "clip_improvement": 0.111457 }, "model.layers.14.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999858, "clip_improvement": 0.250083 }, "model.layers.15.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999876, "clip_improvement": 0.250315 }, "model.layers.15.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954024, "clip_improvement": 0.111581 }, "model.layers.15.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999876, "clip_improvement": 0.25008 }, "model.layers.15.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.955322, "clip_improvement": 0.111896 }, "model.layers.15.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954944, "clip_improvement": 0.111918 }, "model.layers.15.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 33423360, "bits_per_weight": 4.25, "clipped_fraction": 0.954852, "clip_improvement": 0.112017 }, "model.layers.15.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.954248, "clip_improvement": 0.111325 }, "model.layers.16.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.16.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.16.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.954963, "clip_improvement": 0.111801 }, "model.layers.16.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954997, "clip_improvement": 0.111832 }, "model.layers.16.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.95426, "clip_improvement": 0.111103 }, "model.layers.16.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.250263 }, "model.layers.16.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.953856, "clip_improvement": 0.111647 }, "model.layers.16.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999862, "clip_improvement": 0.250095 }, "model.layers.17.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.17.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.17.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955305, "clip_improvement": 0.111751 }, "model.layers.17.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954956, "clip_improvement": 0.112033 }, "model.layers.17.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955562, "clip_improvement": 0.111654 }, "model.layers.17.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999862, "clip_improvement": 0.250546 }, "model.layers.17.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.953981, "clip_improvement": 0.111532 }, "model.layers.17.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999826, "clip_improvement": 0.250273 }, "model.layers.18.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.18.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.18.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955105, "clip_improvement": 0.111719 }, "model.layers.18.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955725, "clip_improvement": 0.111969 }, "model.layers.18.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955257, "clip_improvement": 0.112642 }, "model.layers.18.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999642, "clip_improvement": 0.254122 }, "model.layers.18.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954015, "clip_improvement": 0.111616 }, "model.layers.18.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999813, "clip_improvement": 0.250417 }, "model.layers.19.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999861, "clip_improvement": 0.250504 }, "model.layers.19.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.95399, "clip_improvement": 0.111614 }, "model.layers.19.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999841, "clip_improvement": 0.250121 }, "model.layers.19.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.956494, "clip_improvement": 0.111651 }, "model.layers.19.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.95542, "clip_improvement": 0.111117 }, "model.layers.19.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 33423360, "bits_per_weight": 4.25, "clipped_fraction": 0.954928, "clip_improvement": 0.111848 }, "model.layers.19.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.958228, "clip_improvement": 0.112204 }, "model.layers.2.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.2.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.2.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.95467, "clip_improvement": 0.111766 }, "model.layers.2.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.95437, "clip_improvement": 0.11178 }, "model.layers.2.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955103, "clip_improvement": 0.111672 }, "model.layers.2.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.250175 }, "model.layers.2.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.953527, "clip_improvement": 0.111621 }, "model.layers.2.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999851, "clip_improvement": 0.250174 }, "model.layers.20.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.20.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.20.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.95561, "clip_improvement": 0.111642 }, "model.layers.20.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954899, "clip_improvement": 0.111846 }, "model.layers.20.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.956291, "clip_improvement": 0.111761 }, "model.layers.20.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999859, "clip_improvement": 0.250298 }, "model.layers.20.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954209, "clip_improvement": 0.111617 }, "model.layers.20.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999849, "clip_improvement": 0.250163 }, "model.layers.21.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.21.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.21.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.9551, "clip_improvement": 0.11174 }, "model.layers.21.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954683, "clip_improvement": 0.111484 }, "model.layers.21.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955351, "clip_improvement": 0.111443 }, "model.layers.21.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999832, "clip_improvement": 0.25046 }, "model.layers.21.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.95384, "clip_improvement": 0.111601 }, "model.layers.21.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999833, "clip_improvement": 0.250088 }, "model.layers.22.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.22.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.22.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.954575, "clip_improvement": 0.111636 }, "model.layers.22.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.95437, "clip_improvement": 0.111671 }, "model.layers.22.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955615, "clip_improvement": 0.11214 }, "model.layers.22.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999658, "clip_improvement": 0.254971 }, "model.layers.22.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954316, "clip_improvement": 0.111766 }, "model.layers.22.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999843, "clip_improvement": 0.250311 }, "model.layers.23.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999876, "clip_improvement": 0.250967 }, "model.layers.23.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954176, "clip_improvement": 0.11172 }, "model.layers.23.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999836, "clip_improvement": 0.250167 }, "model.layers.23.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.956104, "clip_improvement": 0.111634 }, "model.layers.23.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954574, "clip_improvement": 0.111474 }, "model.layers.23.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 33423360, "bits_per_weight": 4.25, "clipped_fraction": 0.955151, "clip_improvement": 0.11184 }, "model.layers.23.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.956299, "clip_improvement": 0.111033 }, "model.layers.24.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.24.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.24.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955276, "clip_improvement": 0.111569 }, "model.layers.24.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955094, "clip_improvement": 0.11148 }, "model.layers.24.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954879, "clip_improvement": 0.111599 }, "model.layers.24.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999838, "clip_improvement": 0.250928 }, "model.layers.24.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954313, "clip_improvement": 0.111565 }, "model.layers.24.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999833, "clip_improvement": 0.250264 }, "model.layers.25.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.25.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.25.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.95554, "clip_improvement": 0.111571 }, "model.layers.25.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955497, "clip_improvement": 0.111745 }, "model.layers.25.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954545, "clip_improvement": 0.111737 }, "model.layers.25.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999856, "clip_improvement": 0.250225 }, "model.layers.25.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954248, "clip_improvement": 0.11155 }, "model.layers.25.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999835, "clip_improvement": 0.250287 }, "model.layers.26.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.26.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.26.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955549, "clip_improvement": 0.11143 }, "model.layers.26.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954757, "clip_improvement": 0.111722 }, "model.layers.26.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954948, "clip_improvement": 0.111902 }, "model.layers.26.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999845, "clip_improvement": 0.250635 }, "model.layers.26.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954525, "clip_improvement": 0.111576 }, "model.layers.26.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999866, "clip_improvement": 0.250615 }, "model.layers.27.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999855, "clip_improvement": 0.250789 }, "model.layers.27.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954277, "clip_improvement": 0.111582 }, "model.layers.27.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999861, "clip_improvement": 0.250364 }, "model.layers.27.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.958398, "clip_improvement": 0.111747 }, "model.layers.27.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955628, "clip_improvement": 0.111578 }, "model.layers.27.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 33423360, "bits_per_weight": 4.25, "clipped_fraction": 0.954954, "clip_improvement": 0.111449 }, "model.layers.27.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.956763, "clip_improvement": 0.110793 }, "model.layers.28.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.28.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.28.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.957024, "clip_improvement": 0.111722 }, "model.layers.28.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.95516, "clip_improvement": 0.111571 }, "model.layers.28.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955054, "clip_improvement": 0.111638 }, "model.layers.28.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999862, "clip_improvement": 0.250542 }, "model.layers.28.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954699, "clip_improvement": 0.111631 }, "model.layers.28.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999864, "clip_improvement": 0.250489 }, "model.layers.29.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.29.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.29.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955435, "clip_improvement": 0.111626 }, "model.layers.29.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955644, "clip_improvement": 0.111821 }, "model.layers.29.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955623, "clip_improvement": 0.111556 }, "model.layers.29.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999856, "clip_improvement": 0.250275 }, "model.layers.29.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954318, "clip_improvement": 0.111557 }, "model.layers.29.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999838, "clip_improvement": 0.250411 }, "model.layers.3.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999836, "clip_improvement": 0.250541 }, "model.layers.3.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.953631, "clip_improvement": 0.111623 }, "model.layers.3.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999838, "clip_improvement": 0.249902 }, "model.layers.3.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.954395, "clip_improvement": 0.111428 }, "model.layers.3.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.956112, "clip_improvement": 0.112121 }, "model.layers.3.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 33423360, "bits_per_weight": 4.25, "clipped_fraction": 0.954397, "clip_improvement": 0.111928 }, "model.layers.3.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.954688, "clip_improvement": 0.111859 }, "model.layers.30.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.30.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.30.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.956084, "clip_improvement": 0.111422 }, "model.layers.30.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955347, "clip_improvement": 0.11186 }, "model.layers.30.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.95518, "clip_improvement": 0.111644 }, "model.layers.30.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999835, "clip_improvement": 0.250864 }, "model.layers.30.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954779, "clip_improvement": 0.111442 }, "model.layers.30.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999848, "clip_improvement": 0.25042 }, "model.layers.31.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999815, "clip_improvement": 0.250486 }, "model.layers.31.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954723, "clip_improvement": 0.111373 }, "model.layers.31.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.250653 }, "model.layers.31.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.958081, "clip_improvement": 0.111609 }, "model.layers.31.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954903, "clip_improvement": 0.111542 }, "model.layers.31.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 33423360, "bits_per_weight": 4.25, "clipped_fraction": 0.955337, "clip_improvement": 0.111529 }, "model.layers.31.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.958643, "clip_improvement": 0.110719 }, "model.layers.32.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.32.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.32.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955952, "clip_improvement": 0.111278 }, "model.layers.32.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955221, "clip_improvement": 0.111688 }, "model.layers.32.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954883, "clip_improvement": 0.111769 }, "model.layers.32.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999836, "clip_improvement": 0.250786 }, "model.layers.32.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954603, "clip_improvement": 0.111334 }, "model.layers.32.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999849, "clip_improvement": 0.250684 }, "model.layers.33.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.33.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.33.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.956208, "clip_improvement": 0.11136 }, "model.layers.33.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955294, "clip_improvement": 0.112067 }, "model.layers.33.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955444, "clip_improvement": 0.11159 }, "model.layers.33.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999859, "clip_improvement": 0.250823 }, "model.layers.33.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954881, "clip_improvement": 0.111475 }, "model.layers.33.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999835, "clip_improvement": 0.250689 }, "model.layers.34.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.34.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.34.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955842, "clip_improvement": 0.111378 }, "model.layers.34.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954972, "clip_improvement": 0.112012 }, "model.layers.34.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955933, "clip_improvement": 0.111223 }, "model.layers.34.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999698, "clip_improvement": 0.252964 }, "model.layers.34.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.955424, "clip_improvement": 0.111511 }, "model.layers.34.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999839, "clip_improvement": 0.250805 }, "model.layers.35.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.250783 }, "model.layers.35.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954238, "clip_improvement": 0.11157 }, "model.layers.35.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999845, "clip_improvement": 0.250573 }, "model.layers.35.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.955542, "clip_improvement": 0.111618 }, "model.layers.35.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955241, "clip_improvement": 0.111158 }, "model.layers.35.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 33423360, "bits_per_weight": 4.25, "clipped_fraction": 0.955658, "clip_improvement": 0.111613 }, "model.layers.35.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.957959, "clip_improvement": 0.11173 }, "model.layers.36.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.36.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.36.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955959, "clip_improvement": 0.111665 }, "model.layers.36.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954822, "clip_improvement": 0.111682 }, "model.layers.36.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.956401, "clip_improvement": 0.111882 }, "model.layers.36.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999832, "clip_improvement": 0.250335 }, "model.layers.36.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.953918, "clip_improvement": 0.111382 }, "model.layers.36.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999875, "clip_improvement": 0.250406 }, "model.layers.37.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.37.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.37.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955498, "clip_improvement": 0.111732 }, "model.layers.37.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955689, "clip_improvement": 0.111617 }, "model.layers.37.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.956165, "clip_improvement": 0.111772 }, "model.layers.37.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999848, "clip_improvement": 0.250149 }, "model.layers.37.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954048, "clip_improvement": 0.11159 }, "model.layers.37.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999835, "clip_improvement": 0.25028 }, "model.layers.38.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.38.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.38.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.95522, "clip_improvement": 0.111837 }, "model.layers.38.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954639, "clip_improvement": 0.111934 }, "model.layers.38.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955221, "clip_improvement": 0.111493 }, "model.layers.38.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999841, "clip_improvement": 0.250749 }, "model.layers.38.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954482, "clip_improvement": 0.111663 }, "model.layers.38.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999856, "clip_improvement": 0.250307 }, "model.layers.39.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999881, "clip_improvement": 0.250513 }, "model.layers.39.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954291, "clip_improvement": 0.11161 }, "model.layers.39.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999864, "clip_improvement": 0.250214 }, "model.layers.39.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.955273, "clip_improvement": 0.111974 }, "model.layers.39.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954614, "clip_improvement": 0.111142 }, "model.layers.39.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 33423360, "bits_per_weight": 4.25, "clipped_fraction": 0.954856, "clip_improvement": 0.111827 }, "model.layers.39.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.957812, "clip_improvement": 0.111052 }, "model.layers.4.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.4.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.4.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.954687, "clip_improvement": 0.111505 }, "model.layers.4.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955282, "clip_improvement": 0.112024 }, "model.layers.4.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954976, "clip_improvement": 0.110724 }, "model.layers.4.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999838, "clip_improvement": 0.250218 }, "model.layers.4.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.953931, "clip_improvement": 0.111602 }, "model.layers.4.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999865, "clip_improvement": 0.25003 }, "model.layers.40.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.40.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.40.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955464, "clip_improvement": 0.111769 }, "model.layers.40.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954598, "clip_improvement": 0.111617 }, "model.layers.40.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955359, "clip_improvement": 0.111777 }, "model.layers.40.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999848, "clip_improvement": 0.250317 }, "model.layers.40.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954341, "clip_improvement": 0.111629 }, "model.layers.40.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999864, "clip_improvement": 0.250118 }, "model.layers.41.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.41.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.41.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955395, "clip_improvement": 0.111947 }, "model.layers.41.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955819, "clip_improvement": 0.111906 }, "model.layers.41.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955778, "clip_improvement": 0.111758 }, "model.layers.41.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999866, "clip_improvement": 0.250512 }, "model.layers.41.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954525, "clip_improvement": 0.111461 }, "model.layers.41.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999875, "clip_improvement": 0.250346 }, "model.layers.42.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.42.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.42.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.956357, "clip_improvement": 0.111743 }, "model.layers.42.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.95507, "clip_improvement": 0.111862 }, "model.layers.42.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955485, "clip_improvement": 0.111715 }, "model.layers.42.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999835, "clip_improvement": 0.250647 }, "model.layers.42.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954532, "clip_improvement": 0.111599 }, "model.layers.42.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999868, "clip_improvement": 0.250664 }, "model.layers.43.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999843, "clip_improvement": 0.250459 }, "model.layers.43.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954109, "clip_improvement": 0.111494 }, "model.layers.43.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999868, "clip_improvement": 0.250312 }, "model.layers.43.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.958765, "clip_improvement": 0.111465 }, "model.layers.43.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955729, "clip_improvement": 0.111737 }, "model.layers.43.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 33423360, "bits_per_weight": 4.25, "clipped_fraction": 0.955155, "clip_improvement": 0.111609 }, "model.layers.43.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.955347, "clip_improvement": 0.11167 }, "model.layers.44.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.44.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.44.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955698, "clip_improvement": 0.111134 }, "model.layers.44.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955713, "clip_improvement": 0.111677 }, "model.layers.44.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955037, "clip_improvement": 0.111453 }, "model.layers.44.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.250403 }, "model.layers.44.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.95438, "clip_improvement": 0.111442 }, "model.layers.44.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999871, "clip_improvement": 0.250432 }, "model.layers.45.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.45.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.45.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955984, "clip_improvement": 0.111564 }, "model.layers.45.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954919, "clip_improvement": 0.111606 }, "model.layers.45.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955033, "clip_improvement": 0.111745 }, "model.layers.45.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999836, "clip_improvement": 0.250448 }, "model.layers.45.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954606, "clip_improvement": 0.111444 }, "model.layers.45.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999872, "clip_improvement": 0.25047 }, "model.layers.46.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.46.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.46.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955395, "clip_improvement": 0.111414 }, "model.layers.46.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955245, "clip_improvement": 0.11177 }, "model.layers.46.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955282, "clip_improvement": 0.11177 }, "model.layers.46.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999831, "clip_improvement": 0.25042 }, "model.layers.46.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954372, "clip_improvement": 0.111566 }, "model.layers.46.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999831, "clip_improvement": 0.250489 }, "model.layers.47.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999833, "clip_improvement": 0.250788 }, "model.layers.47.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954193, "clip_improvement": 0.111599 }, "model.layers.47.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999865, "clip_improvement": 0.250492 }, "model.layers.47.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.955835, "clip_improvement": 0.111922 }, "model.layers.47.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.95612, "clip_improvement": 0.111773 }, "model.layers.47.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 33423360, "bits_per_weight": 4.25, "clipped_fraction": 0.955347, "clip_improvement": 0.111727 }, "model.layers.47.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.955396, "clip_improvement": 0.111981 }, "model.layers.48.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.48.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.48.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955693, "clip_improvement": 0.111392 }, "model.layers.48.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955212, "clip_improvement": 0.111621 }, "model.layers.48.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955648, "clip_improvement": 0.111403 }, "model.layers.48.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999861, "clip_improvement": 0.25088 }, "model.layers.48.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954881, "clip_improvement": 0.111585 }, "model.layers.48.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999846, "clip_improvement": 0.250546 }, "model.layers.49.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.49.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.49.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955832, "clip_improvement": 0.111591 }, "model.layers.49.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954622, "clip_improvement": 0.111646 }, "model.layers.49.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955746, "clip_improvement": 0.111534 }, "model.layers.49.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999845, "clip_improvement": 0.250922 }, "model.layers.49.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954422, "clip_improvement": 0.111833 }, "model.layers.49.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999856, "clip_improvement": 0.250509 }, "model.layers.5.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.5.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.5.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.954868, "clip_improvement": 0.111796 }, "model.layers.5.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954374, "clip_improvement": 0.111633 }, "model.layers.5.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955383, "clip_improvement": 0.111338 }, "model.layers.5.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999833, "clip_improvement": 0.250193 }, "model.layers.5.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954343, "clip_improvement": 0.111765 }, "model.layers.5.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999852, "clip_improvement": 0.25008 }, "model.layers.50.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.50.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.50.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955898, "clip_improvement": 0.111568 }, "model.layers.50.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955184, "clip_improvement": 0.111782 }, "model.layers.50.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955221, "clip_improvement": 0.11127 }, "model.layers.50.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999796, "clip_improvement": 0.252372 }, "model.layers.50.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954739, "clip_improvement": 0.111631 }, "model.layers.50.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999845, "clip_improvement": 0.25089 }, "model.layers.51.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999829, "clip_improvement": 0.251376 }, "model.layers.51.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954689, "clip_improvement": 0.111677 }, "model.layers.51.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999845, "clip_improvement": 0.250395 }, "model.layers.51.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.956372, "clip_improvement": 0.112017 }, "model.layers.51.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.956498, "clip_improvement": 0.110943 }, "model.layers.51.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 33423360, "bits_per_weight": 4.25, "clipped_fraction": 0.955149, "clip_improvement": 0.111261 }, "model.layers.51.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.95835, "clip_improvement": 0.111622 }, "model.layers.52.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.52.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.52.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955757, "clip_improvement": 0.111749 }, "model.layers.52.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955192, "clip_improvement": 0.111994 }, "model.layers.52.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955644, "clip_improvement": 0.111805 }, "model.layers.52.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999848, "clip_improvement": 0.250889 }, "model.layers.52.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954677, "clip_improvement": 0.111768 }, "model.layers.52.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999868, "clip_improvement": 0.250569 }, "model.layers.53.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.53.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.53.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955112, "clip_improvement": 0.111929 }, "model.layers.53.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954875, "clip_improvement": 0.111492 }, "model.layers.53.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955408, "clip_improvement": 0.111976 }, "model.layers.53.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999828, "clip_improvement": 0.251118 }, "model.layers.53.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954708, "clip_improvement": 0.111873 }, "model.layers.53.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999845, "clip_improvement": 0.250399 }, "model.layers.54.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.54.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.54.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999778, "clip_improvement": 0.250885 }, "model.layers.54.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954549, "clip_improvement": 0.111832 }, "model.layers.54.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.95529, "clip_improvement": 0.112607 }, "model.layers.54.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999819, "clip_improvement": 0.25104 }, "model.layers.54.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.955171, "clip_improvement": 0.111786 }, "model.layers.54.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999835, "clip_improvement": 0.250397 }, "model.layers.55.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999864, "clip_improvement": 0.250581 }, "model.layers.55.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954804, "clip_improvement": 0.111901 }, "model.layers.55.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999802, "clip_improvement": 0.250183 }, "model.layers.55.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.959204, "clip_improvement": 0.111035 }, "model.layers.55.self_attn.o_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999801, "clip_improvement": 0.252846 }, "model.layers.55.self_attn.q_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25559040, "bits_per_weight": 3.25, "clipped_fraction": 0.999866, "clip_improvement": 0.250785 }, "model.layers.55.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.957275, "clip_improvement": 0.110343 }, "model.layers.56.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.56.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.56.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955205, "clip_improvement": 0.112076 }, "model.layers.56.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954736, "clip_improvement": 0.111756 }, "model.layers.56.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955355, "clip_improvement": 0.112062 }, "model.layers.56.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999868, "clip_improvement": 0.250452 }, "model.layers.56.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954389, "clip_improvement": 0.111774 }, "model.layers.56.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999841, "clip_improvement": 0.250318 }, "model.layers.57.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.57.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.57.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955303, "clip_improvement": 0.112053 }, "model.layers.57.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954313, "clip_improvement": 0.111759 }, "model.layers.57.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955802, "clip_improvement": 0.112172 }, "model.layers.57.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999841, "clip_improvement": 0.250379 }, "model.layers.57.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954439, "clip_improvement": 0.111959 }, "model.layers.57.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999874, "clip_improvement": 0.250097 }, "model.layers.58.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.58.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.58.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999814, "clip_improvement": 0.250903 }, "model.layers.58.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.99989, "clip_improvement": 0.250319 }, "model.layers.58.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999829, "clip_improvement": 0.25211 }, "model.layers.58.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.99981, "clip_improvement": 0.250733 }, "model.layers.58.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954584, "clip_improvement": 0.11205 }, "model.layers.58.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999856, "clip_improvement": 0.250113 }, "model.layers.59.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999864, "clip_improvement": 0.250645 }, "model.layers.59.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999848, "clip_improvement": 0.25073 }, "model.layers.59.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999821, "clip_improvement": 0.250441 }, "model.layers.59.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.959399, "clip_improvement": 0.110837 }, "model.layers.59.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955957, "clip_improvement": 0.111146 }, "model.layers.59.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 33423360, "bits_per_weight": 4.25, "clipped_fraction": 0.954901, "clip_improvement": 0.111785 }, "model.layers.59.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.957861, "clip_improvement": 0.113004 }, "model.layers.6.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.6.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.6.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955254, "clip_improvement": 0.1117 }, "model.layers.6.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.95518, "clip_improvement": 0.111621 }, "model.layers.6.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955542, "clip_improvement": 0.112362 }, "model.layers.6.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999703, "clip_improvement": 0.252978 }, "model.layers.6.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.953823, "clip_improvement": 0.111815 }, "model.layers.6.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999835, "clip_improvement": 0.250559 }, "model.layers.60.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.60.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.60.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955249, "clip_improvement": 0.112038 }, "model.layers.60.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954533, "clip_improvement": 0.111869 }, "model.layers.60.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955074, "clip_improvement": 0.111918 }, "model.layers.60.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999843, "clip_improvement": 0.250864 }, "model.layers.60.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.95441, "clip_improvement": 0.111778 }, "model.layers.60.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999849, "clip_improvement": 0.250258 }, "model.layers.61.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.61.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.61.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955271, "clip_improvement": 0.111855 }, "model.layers.61.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955009, "clip_improvement": 0.112015 }, "model.layers.61.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955607, "clip_improvement": 0.111875 }, "model.layers.61.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999843, "clip_improvement": 0.250382 }, "model.layers.61.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954732, "clip_improvement": 0.111989 }, "model.layers.61.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999822, "clip_improvement": 0.250265 }, "model.layers.62.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.62.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.62.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955854, "clip_improvement": 0.111737 }, "model.layers.62.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954631, "clip_improvement": 0.112326 }, "model.layers.62.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.956116, "clip_improvement": 0.111719 }, "model.layers.62.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999843, "clip_improvement": 0.251275 }, "model.layers.62.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954667, "clip_improvement": 0.111817 }, "model.layers.62.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999818, "clip_improvement": 0.250395 }, "model.layers.63.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999875, "clip_improvement": 0.252275 }, "model.layers.63.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954699, "clip_improvement": 0.111917 }, "model.layers.63.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999833, "clip_improvement": 0.250269 }, "model.layers.63.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.959839, "clip_improvement": 0.111721 }, "model.layers.63.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.959017, "clip_improvement": 0.110433 }, "model.layers.63.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 33423360, "bits_per_weight": 4.25, "clipped_fraction": 0.955326, "clip_improvement": 0.111927 }, "model.layers.63.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.955786, "clip_improvement": 0.112904 }, "model.layers.7.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999831, "clip_improvement": 0.250414 }, "model.layers.7.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.954307, "clip_improvement": 0.111636 }, "model.layers.7.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999843, "clip_improvement": 0.250154 }, "model.layers.7.self_attn.k_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.958765, "clip_improvement": 0.112238 }, "model.layers.7.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954407, "clip_improvement": 0.111939 }, "model.layers.7.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 33423360, "bits_per_weight": 4.25, "clipped_fraction": 0.955052, "clip_improvement": 0.111794 }, "model.layers.7.self_attn.v_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2785280, "bits_per_weight": 4.25, "clipped_fraction": 0.956201, "clip_improvement": 0.112123 }, "model.layers.8.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.8.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.8.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955012, "clip_improvement": 0.111602 }, "model.layers.8.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954907, "clip_improvement": 0.111852 }, "model.layers.8.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954948, "clip_improvement": 0.111565 }, "model.layers.8.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999869, "clip_improvement": 0.250149 }, "model.layers.8.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.953932, "clip_improvement": 0.111709 }, "model.layers.8.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.249985 }, "model.layers.9.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.9.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.9.linear_attn.in_proj_qkv": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 27852800, "bits_per_weight": 4.25, "clipped_fraction": 0.955198, "clip_improvement": 0.112071 }, "model.layers.9.linear_attn.in_proj_z": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.955151, "clip_improvement": 0.112055 }, "model.layers.9.linear_attn.out_proj": { "bits": 4, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 16711680, "bits_per_weight": 4.25, "clipped_fraction": 0.954606, "clip_improvement": 0.111772 }, "model.layers.9.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999829, "clip_improvement": 0.250178 }, "model.layers.9.mlp.gate_proj": { "bits": 4, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 47349760, "bits_per_weight": 4.25, "clipped_fraction": 0.953882, "clip_improvement": 0.111555 }, "model.layers.9.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999842, "clip_improvement": 0.250002 } } }