{ "schema": "dynquant_checkpoint_v2", "dynquant_core": "0.5.2", "quantization_config": { "quant_method": "dynquant", "checkpoint_format": "dynquant-packed", "schema_version": 1, "version": "0.5.2", "group_size": 128, "symmetric": false, "lm_head_quantized": true, "modules_to_not_convert": [], "modules": { "lm_head": { "bits": 3, "out_features": 248320 }, "model.embed_tokens": { "bits": 3, "out_features": 248320 }, "model.layers.0.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.0.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.0.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.0.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.0.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.0.mlp.down_proj": { "bits": 2, "out_features": 5120 }, "model.layers.0.mlp.gate_proj": { "bits": 2, "out_features": 17408 }, "model.layers.0.mlp.up_proj": { "bits": 2, "out_features": 17408 }, "model.layers.1.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.1.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.1.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.1.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.1.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.1.mlp.down_proj": { "bits": 2, "out_features": 5120 }, "model.layers.1.mlp.gate_proj": { "bits": 2, "out_features": 17408 }, "model.layers.1.mlp.up_proj": { "bits": 2, "out_features": 17408 }, "model.layers.10.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.10.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.10.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.10.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.10.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.10.mlp.down_proj": { "bits": 2, "out_features": 5120 }, "model.layers.10.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.10.mlp.up_proj": { "bits": 2, "out_features": 17408 }, "model.layers.11.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.11.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.11.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.11.self_attn.k_proj": { "bits": 3, "out_features": 1024 }, "model.layers.11.self_attn.o_proj": { "bits": 3, "out_features": 5120 }, "model.layers.11.self_attn.q_proj": { "bits": 3, "out_features": 12288 }, "model.layers.11.self_attn.v_proj": { "bits": 3, "out_features": 1024 }, "model.layers.12.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.12.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.12.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.12.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.12.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.12.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.12.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.12.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.13.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.13.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.13.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.13.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.13.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.13.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.13.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.13.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.14.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.14.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.14.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.14.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.14.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.14.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.14.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.14.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.15.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.15.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.15.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.15.self_attn.k_proj": { "bits": 3, "out_features": 1024 }, "model.layers.15.self_attn.o_proj": { "bits": 3, "out_features": 5120 }, "model.layers.15.self_attn.q_proj": { "bits": 3, "out_features": 12288 }, "model.layers.15.self_attn.v_proj": { "bits": 3, "out_features": 1024 }, "model.layers.16.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.16.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.16.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.16.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.16.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.16.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.16.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.16.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.17.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.17.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.17.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.17.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.17.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.17.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.17.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.17.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.18.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.18.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.18.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.18.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.18.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.18.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.18.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.18.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.19.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.19.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.19.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.19.self_attn.k_proj": { "bits": 3, "out_features": 1024 }, "model.layers.19.self_attn.o_proj": { "bits": 3, "out_features": 5120 }, "model.layers.19.self_attn.q_proj": { "bits": 3, "out_features": 12288 }, "model.layers.19.self_attn.v_proj": { "bits": 3, "out_features": 1024 }, "model.layers.2.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.2.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.2.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.2.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.2.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.2.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.2.mlp.gate_proj": { "bits": 2, "out_features": 17408 }, "model.layers.2.mlp.up_proj": { "bits": 2, "out_features": 17408 }, "model.layers.20.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.20.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.20.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.20.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.20.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.20.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.20.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.20.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.21.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.21.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.21.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.21.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.21.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.21.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.21.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.21.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.22.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.22.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.22.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.22.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.22.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.22.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.22.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.22.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.23.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.23.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.23.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.23.self_attn.k_proj": { "bits": 3, "out_features": 1024 }, "model.layers.23.self_attn.o_proj": { "bits": 3, "out_features": 5120 }, "model.layers.23.self_attn.q_proj": { "bits": 3, "out_features": 12288 }, "model.layers.23.self_attn.v_proj": { "bits": 3, "out_features": 1024 }, "model.layers.24.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.24.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.24.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.24.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.24.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.24.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.24.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.24.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.25.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.25.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.25.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.25.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.25.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.25.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.25.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.25.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.26.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.26.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.26.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.26.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.26.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.26.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.26.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.26.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.27.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.27.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.27.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.27.self_attn.k_proj": { "bits": 3, "out_features": 1024 }, "model.layers.27.self_attn.o_proj": { "bits": 3, "out_features": 5120 }, "model.layers.27.self_attn.q_proj": { "bits": 3, "out_features": 12288 }, "model.layers.27.self_attn.v_proj": { "bits": 3, "out_features": 1024 }, "model.layers.28.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.28.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.28.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.28.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.28.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.28.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.28.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.28.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.29.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.29.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.29.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.29.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.29.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.29.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.29.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.29.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.3.mlp.down_proj": { "bits": 2, "out_features": 5120 }, "model.layers.3.mlp.gate_proj": { "bits": 2, "out_features": 17408 }, "model.layers.3.mlp.up_proj": { "bits": 2, "out_features": 17408 }, "model.layers.3.self_attn.k_proj": { "bits": 3, "out_features": 1024 }, "model.layers.3.self_attn.o_proj": { "bits": 3, "out_features": 5120 }, "model.layers.3.self_attn.q_proj": { "bits": 3, "out_features": 12288 }, "model.layers.3.self_attn.v_proj": { "bits": 3, "out_features": 1024 }, "model.layers.30.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.30.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.30.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.30.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.30.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.30.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.30.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.30.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.31.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.31.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.31.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.31.self_attn.k_proj": { "bits": 3, "out_features": 1024 }, "model.layers.31.self_attn.o_proj": { "bits": 3, "out_features": 5120 }, "model.layers.31.self_attn.q_proj": { "bits": 3, "out_features": 12288 }, "model.layers.31.self_attn.v_proj": { "bits": 3, "out_features": 1024 }, "model.layers.32.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.32.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.32.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.32.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.32.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.32.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.32.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.32.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.33.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.33.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.33.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.33.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.33.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.33.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.33.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.33.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.34.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.34.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.34.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.34.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.34.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.34.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.34.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.34.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.35.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.35.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.35.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.35.self_attn.k_proj": { "bits": 3, "out_features": 1024 }, "model.layers.35.self_attn.o_proj": { "bits": 3, "out_features": 5120 }, "model.layers.35.self_attn.q_proj": { "bits": 3, "out_features": 12288 }, "model.layers.35.self_attn.v_proj": { "bits": 3, "out_features": 1024 }, "model.layers.36.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.36.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.36.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.36.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.36.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.36.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.36.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.36.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.37.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.37.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.37.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.37.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.37.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.37.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.37.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.37.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.38.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.38.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.38.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.38.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.38.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.38.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.38.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.38.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.39.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.39.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.39.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.39.self_attn.k_proj": { "bits": 3, "out_features": 1024 }, "model.layers.39.self_attn.o_proj": { "bits": 3, "out_features": 5120 }, "model.layers.39.self_attn.q_proj": { "bits": 3, "out_features": 12288 }, "model.layers.39.self_attn.v_proj": { "bits": 3, "out_features": 1024 }, "model.layers.4.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.4.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.4.linear_attn.in_proj_qkv": { "bits": 2, "out_features": 10240 }, "model.layers.4.linear_attn.in_proj_z": { "bits": 2, "out_features": 6144 }, "model.layers.4.linear_attn.out_proj": { "bits": 2, "out_features": 5120 }, "model.layers.4.mlp.down_proj": { "bits": 2, "out_features": 5120 }, "model.layers.4.mlp.gate_proj": { "bits": 2, "out_features": 17408 }, "model.layers.4.mlp.up_proj": { "bits": 2, "out_features": 17408 }, "model.layers.40.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.40.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.40.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.40.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.40.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.40.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.40.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.40.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.41.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.41.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.41.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.41.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.41.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.41.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.41.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.41.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.42.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.42.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.42.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.42.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.42.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.42.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.42.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.42.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.43.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.43.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.43.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.43.self_attn.k_proj": { "bits": 3, "out_features": 1024 }, "model.layers.43.self_attn.o_proj": { "bits": 3, "out_features": 5120 }, "model.layers.43.self_attn.q_proj": { "bits": 3, "out_features": 12288 }, "model.layers.43.self_attn.v_proj": { "bits": 3, "out_features": 1024 }, "model.layers.44.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.44.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.44.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.44.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.44.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.44.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.44.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.44.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.45.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.45.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.45.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.45.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.45.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.45.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.45.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.45.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.46.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.46.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.46.linear_attn.in_proj_qkv": { "bits": 2, "out_features": 10240 }, "model.layers.46.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.46.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.46.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.46.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.46.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.47.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.47.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.47.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.47.self_attn.k_proj": { "bits": 2, "out_features": 1024 }, "model.layers.47.self_attn.o_proj": { "bits": 2, "out_features": 5120 }, "model.layers.47.self_attn.q_proj": { "bits": 2, "out_features": 12288 }, "model.layers.47.self_attn.v_proj": { "bits": 3, "out_features": 1024 }, "model.layers.48.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.48.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.48.linear_attn.in_proj_qkv": { "bits": 2, "out_features": 10240 }, "model.layers.48.linear_attn.in_proj_z": { "bits": 2, "out_features": 6144 }, "model.layers.48.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.48.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.48.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.48.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.49.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.49.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.49.linear_attn.in_proj_qkv": { "bits": 2, "out_features": 10240 }, "model.layers.49.linear_attn.in_proj_z": { "bits": 2, "out_features": 6144 }, "model.layers.49.linear_attn.out_proj": { "bits": 2, "out_features": 5120 }, "model.layers.49.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.49.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.49.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.5.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.5.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.5.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.5.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.5.linear_attn.out_proj": { "bits": 2, "out_features": 5120 }, "model.layers.5.mlp.down_proj": { "bits": 2, "out_features": 5120 }, "model.layers.5.mlp.gate_proj": { "bits": 2, "out_features": 17408 }, "model.layers.5.mlp.up_proj": { "bits": 2, "out_features": 17408 }, "model.layers.50.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.50.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.50.linear_attn.in_proj_qkv": { "bits": 2, "out_features": 10240 }, "model.layers.50.linear_attn.in_proj_z": { "bits": 2, "out_features": 6144 }, "model.layers.50.linear_attn.out_proj": { "bits": 2, "out_features": 5120 }, "model.layers.50.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.50.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.50.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.51.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.51.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.51.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.51.self_attn.k_proj": { "bits": 2, "out_features": 1024 }, "model.layers.51.self_attn.o_proj": { "bits": 2, "out_features": 5120 }, "model.layers.51.self_attn.q_proj": { "bits": 2, "out_features": 12288 }, "model.layers.51.self_attn.v_proj": { "bits": 2, "out_features": 1024 }, "model.layers.52.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.52.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.52.linear_attn.in_proj_qkv": { "bits": 2, "out_features": 10240 }, "model.layers.52.linear_attn.in_proj_z": { "bits": 2, "out_features": 6144 }, "model.layers.52.linear_attn.out_proj": { "bits": 2, "out_features": 5120 }, "model.layers.52.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.52.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.52.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.53.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.53.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.53.linear_attn.in_proj_qkv": { "bits": 2, "out_features": 10240 }, "model.layers.53.linear_attn.in_proj_z": { "bits": 2, "out_features": 6144 }, "model.layers.53.linear_attn.out_proj": { "bits": 2, "out_features": 5120 }, "model.layers.53.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.53.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.53.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.54.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.54.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.54.linear_attn.in_proj_qkv": { "bits": 2, "out_features": 10240 }, "model.layers.54.linear_attn.in_proj_z": { "bits": 2, "out_features": 6144 }, "model.layers.54.linear_attn.out_proj": { "bits": 2, "out_features": 5120 }, "model.layers.54.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.54.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.54.mlp.up_proj": { "bits": 2, "out_features": 17408 }, "model.layers.55.mlp.down_proj": { "bits": 2, "out_features": 5120 }, "model.layers.55.mlp.gate_proj": { "bits": 2, "out_features": 17408 }, "model.layers.55.mlp.up_proj": { "bits": 2, "out_features": 17408 }, "model.layers.55.self_attn.k_proj": { "bits": 2, "out_features": 1024 }, "model.layers.55.self_attn.o_proj": { "bits": 2, "out_features": 5120 }, "model.layers.55.self_attn.q_proj": { "bits": 2, "out_features": 12288 }, "model.layers.55.self_attn.v_proj": { "bits": 2, "out_features": 1024 }, "model.layers.56.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.56.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.56.linear_attn.in_proj_qkv": { "bits": 2, "out_features": 10240 }, "model.layers.56.linear_attn.in_proj_z": { "bits": 2, "out_features": 6144 }, "model.layers.56.linear_attn.out_proj": { "bits": 2, "out_features": 5120 }, "model.layers.56.mlp.down_proj": { "bits": 2, "out_features": 5120 }, "model.layers.56.mlp.gate_proj": { "bits": 2, "out_features": 17408 }, "model.layers.56.mlp.up_proj": { "bits": 2, "out_features": 17408 }, "model.layers.57.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.57.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.57.linear_attn.in_proj_qkv": { "bits": 2, "out_features": 10240 }, "model.layers.57.linear_attn.in_proj_z": { "bits": 2, "out_features": 6144 }, "model.layers.57.linear_attn.out_proj": { "bits": 2, "out_features": 5120 }, "model.layers.57.mlp.down_proj": { "bits": 2, "out_features": 5120 }, "model.layers.57.mlp.gate_proj": { "bits": 2, "out_features": 17408 }, "model.layers.57.mlp.up_proj": { "bits": 2, "out_features": 17408 }, "model.layers.58.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.58.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.58.linear_attn.in_proj_qkv": { "bits": 2, "out_features": 10240 }, "model.layers.58.linear_attn.in_proj_z": { "bits": 2, "out_features": 6144 }, "model.layers.58.linear_attn.out_proj": { "bits": 2, "out_features": 5120 }, "model.layers.58.mlp.down_proj": { "bits": 2, "out_features": 5120 }, "model.layers.58.mlp.gate_proj": { "bits": 2, "out_features": 17408 }, "model.layers.58.mlp.up_proj": { "bits": 2, "out_features": 17408 }, "model.layers.59.mlp.down_proj": { "bits": 2, "out_features": 5120 }, "model.layers.59.mlp.gate_proj": { "bits": 2, "out_features": 17408 }, "model.layers.59.mlp.up_proj": { "bits": 2, "out_features": 17408 }, "model.layers.59.self_attn.k_proj": { "bits": 2, "out_features": 1024 }, "model.layers.59.self_attn.o_proj": { "bits": 2, "out_features": 5120 }, "model.layers.59.self_attn.q_proj": { "bits": 2, "out_features": 12288 }, "model.layers.59.self_attn.v_proj": { "bits": 2, "out_features": 1024 }, "model.layers.6.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.6.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.6.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.6.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.6.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.6.mlp.down_proj": { "bits": 2, "out_features": 5120 }, "model.layers.6.mlp.gate_proj": { "bits": 3, "out_features": 17408 }, "model.layers.6.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.60.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.60.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.60.linear_attn.in_proj_qkv": { "bits": 2, "out_features": 10240 }, "model.layers.60.linear_attn.in_proj_z": { "bits": 2, "out_features": 6144 }, "model.layers.60.linear_attn.out_proj": { "bits": 2, "out_features": 5120 }, "model.layers.60.mlp.down_proj": { "bits": 2, "out_features": 5120 }, "model.layers.60.mlp.gate_proj": { "bits": 2, "out_features": 17408 }, "model.layers.60.mlp.up_proj": { "bits": 2, "out_features": 17408 }, "model.layers.61.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.61.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.61.linear_attn.in_proj_qkv": { "bits": 2, "out_features": 10240 }, "model.layers.61.linear_attn.in_proj_z": { "bits": 2, "out_features": 6144 }, "model.layers.61.linear_attn.out_proj": { "bits": 2, "out_features": 5120 }, "model.layers.61.mlp.down_proj": { "bits": 2, "out_features": 5120 }, "model.layers.61.mlp.gate_proj": { "bits": 2, "out_features": 17408 }, "model.layers.61.mlp.up_proj": { "bits": 2, "out_features": 17408 }, "model.layers.62.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.62.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.62.linear_attn.in_proj_qkv": { "bits": 2, "out_features": 10240 }, "model.layers.62.linear_attn.in_proj_z": { "bits": 2, "out_features": 6144 }, "model.layers.62.linear_attn.out_proj": { "bits": 2, "out_features": 5120 }, "model.layers.62.mlp.down_proj": { "bits": 2, "out_features": 5120 }, "model.layers.62.mlp.gate_proj": { "bits": 2, "out_features": 17408 }, "model.layers.62.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.63.mlp.down_proj": { "bits": 3, "out_features": 5120 }, "model.layers.63.mlp.gate_proj": { "bits": 2, "out_features": 17408 }, "model.layers.63.mlp.up_proj": { "bits": 3, "out_features": 17408 }, "model.layers.63.self_attn.k_proj": { "bits": 3, "out_features": 1024 }, "model.layers.63.self_attn.o_proj": { "bits": 2, "out_features": 5120 }, "model.layers.63.self_attn.q_proj": { "bits": 2, "out_features": 12288 }, "model.layers.63.self_attn.v_proj": { "bits": 2, "out_features": 1024 }, "model.layers.7.mlp.down_proj": { "bits": 2, "out_features": 5120 }, "model.layers.7.mlp.gate_proj": { "bits": 2, "out_features": 17408 }, "model.layers.7.mlp.up_proj": { "bits": 2, "out_features": 17408 }, "model.layers.7.self_attn.k_proj": { "bits": 3, "out_features": 1024 }, "model.layers.7.self_attn.o_proj": { "bits": 3, "out_features": 5120 }, "model.layers.7.self_attn.q_proj": { "bits": 3, "out_features": 12288 }, "model.layers.7.self_attn.v_proj": { "bits": 3, "out_features": 1024 }, "model.layers.8.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.8.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.8.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.8.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.8.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.8.mlp.down_proj": { "bits": 2, "out_features": 5120 }, "model.layers.8.mlp.gate_proj": { "bits": 2, "out_features": 17408 }, "model.layers.8.mlp.up_proj": { "bits": 2, "out_features": 17408 }, "model.layers.9.linear_attn.in_proj_a": { "bits": 8, "out_features": 48 }, "model.layers.9.linear_attn.in_proj_b": { "bits": 8, "out_features": 48 }, "model.layers.9.linear_attn.in_proj_qkv": { "bits": 3, "out_features": 10240 }, "model.layers.9.linear_attn.in_proj_z": { "bits": 3, "out_features": 6144 }, "model.layers.9.linear_attn.out_proj": { "bits": 3, "out_features": 5120 }, "model.layers.9.mlp.down_proj": { "bits": 2, "out_features": 5120 }, "model.layers.9.mlp.gate_proj": { "bits": 2, "out_features": 17408 }, "model.layers.9.mlp.up_proj": { "bits": 2, "out_features": 17408 } } }, "files": [ "model-00001-of-00003.safetensors", "model-00002-of-00003.safetensors", "model-00003-of-00003.safetensors", "model.safetensors.index.json" ], "accounting": { "packed_nbytes": 10080337920, "dense_nbytes": 5291008, "total_nbytes": 10085628928, "quantized_elements": 26893352960, "average_bits": 2.998611 }, "tied": {}, "expert_banks": [], "provenance": { "dynquant_core": "0.5.2", "model": "/workspace/runs-s2/qwen38-27b.text2sql/merged", "stats": "/workspace/runs-s2/qwen38-27b.text2sql/stats/dynquant_stats.json", "allocator": "rank_product", "widths_from": "3.00 avg bits", "predicted_nbytes": 10085628928 }, "layers": { "lm_head": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 248320, 5120 ], "num_rows": 248320, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 516505600, "bits_per_weight": 3.25, "clipped_fraction": 0.999805, "clip_improvement": 0.250989 }, "model.embed_tokens": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 248320, 5120 ], "num_rows": 248320, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 516505600, "bits_per_weight": 3.25, "clipped_fraction": 0.999813, "clip_improvement": 0.250535 }, "model.layers.0.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.0.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.0.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999844, "clip_improvement": 0.250091 }, "model.layers.0.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999849, "clip_improvement": 0.250607 }, "model.layers.0.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999845, "clip_improvement": 0.249668 }, "model.layers.0.mlp.down_proj": { "bits": 2, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999987, "clip_improvement": 0.353846 }, "model.layers.0.mlp.gate_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999868, "clip_improvement": 0.353318 }, "model.layers.0.mlp.up_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999978, "clip_improvement": 0.353266 }, "model.layers.1.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.1.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.1.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999831, "clip_improvement": 0.250704 }, "model.layers.1.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.99987, "clip_improvement": 0.250891 }, "model.layers.1.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999837, "clip_improvement": 0.250937 }, "model.layers.1.mlp.down_proj": { "bits": 2, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999721, "clip_improvement": 0.354811 }, "model.layers.1.mlp.gate_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999894, "clip_improvement": 0.353783 }, "model.layers.1.mlp.up_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999967, "clip_improvement": 0.353796 }, "model.layers.10.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.10.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.10.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999858, "clip_improvement": 0.250787 }, "model.layers.10.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999849, "clip_improvement": 0.250581 }, "model.layers.10.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999821, "clip_improvement": 0.250037 }, "model.layers.10.mlp.down_proj": { "bits": 2, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999879, "clip_improvement": 0.353552 }, "model.layers.10.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999825, "clip_improvement": 0.250361 }, "model.layers.10.mlp.up_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999994, "clip_improvement": 0.353712 }, "model.layers.11.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999842, "clip_improvement": 0.250375 }, "model.layers.11.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999864, "clip_improvement": 0.250154 }, "model.layers.11.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999866, "clip_improvement": 0.24994 }, "model.layers.11.self_attn.k_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2129920, "bits_per_weight": 3.25, "clipped_fraction": 0.999927, "clip_improvement": 0.253593 }, "model.layers.11.self_attn.o_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999849, "clip_improvement": 0.250674 }, "model.layers.11.self_attn.q_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25559040, "bits_per_weight": 3.25, "clipped_fraction": 0.999827, "clip_improvement": 0.251097 }, "model.layers.11.self_attn.v_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2129920, "bits_per_weight": 3.25, "clipped_fraction": 0.999829, "clip_improvement": 0.251107 }, "model.layers.12.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.12.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.12.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.99988, "clip_improvement": 0.250457 }, "model.layers.12.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999813, "clip_improvement": 0.250526 }, "model.layers.12.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999825, "clip_improvement": 0.250706 }, "model.layers.12.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999842, "clip_improvement": 0.250405 }, "model.layers.12.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999845, "clip_improvement": 0.250209 }, "model.layers.12.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999832, "clip_improvement": 0.249959 }, "model.layers.13.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.13.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.13.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999875, "clip_improvement": 0.250257 }, "model.layers.13.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.99989, "clip_improvement": 0.250167 }, "model.layers.13.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999849, "clip_improvement": 0.250784 }, "model.layers.13.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999806, "clip_improvement": 0.249997 }, "model.layers.13.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999836, "clip_improvement": 0.249919 }, "model.layers.13.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999848, "clip_improvement": 0.25019 }, "model.layers.14.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.14.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.14.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999871, "clip_improvement": 0.250805 }, "model.layers.14.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999805, "clip_improvement": 0.250627 }, "model.layers.14.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999845, "clip_improvement": 0.250846 }, "model.layers.14.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999855, "clip_improvement": 0.250201 }, "model.layers.14.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999861, "clip_improvement": 0.249788 }, "model.layers.14.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999858, "clip_improvement": 0.250083 }, "model.layers.15.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999876, "clip_improvement": 0.250315 }, "model.layers.15.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999856, "clip_improvement": 0.249957 }, "model.layers.15.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999876, "clip_improvement": 0.25008 }, "model.layers.15.self_attn.k_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2129920, "bits_per_weight": 3.25, "clipped_fraction": 0.999902, "clip_improvement": 0.252909 }, "model.layers.15.self_attn.o_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999841, "clip_improvement": 0.250455 }, "model.layers.15.self_attn.q_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25559040, "bits_per_weight": 3.25, "clipped_fraction": 0.99978, "clip_improvement": 0.25103 }, "model.layers.15.self_attn.v_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2129920, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.251013 }, "model.layers.16.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.16.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.16.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999846, "clip_improvement": 0.250546 }, "model.layers.16.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999858, "clip_improvement": 0.250616 }, "model.layers.16.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999906, "clip_improvement": 0.25027 }, "model.layers.16.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.250263 }, "model.layers.16.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999828, "clip_improvement": 0.250176 }, "model.layers.16.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999862, "clip_improvement": 0.250095 }, "model.layers.17.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.17.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.17.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999834, "clip_improvement": 0.250565 }, "model.layers.17.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999866, "clip_improvement": 0.250324 }, "model.layers.17.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999841, "clip_improvement": 0.250641 }, "model.layers.17.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999862, "clip_improvement": 0.250546 }, "model.layers.17.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999829, "clip_improvement": 0.250166 }, "model.layers.17.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999826, "clip_improvement": 0.250273 }, "model.layers.18.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.18.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.18.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.99979, "clip_improvement": 0.250826 }, "model.layers.18.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999862, "clip_improvement": 0.250157 }, "model.layers.18.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999849, "clip_improvement": 0.251954 }, "model.layers.18.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999642, "clip_improvement": 0.254122 }, "model.layers.18.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999809, "clip_improvement": 0.250923 }, "model.layers.18.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999813, "clip_improvement": 0.250417 }, "model.layers.19.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999861, "clip_improvement": 0.250504 }, "model.layers.19.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999856, "clip_improvement": 0.250005 }, "model.layers.19.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999841, "clip_improvement": 0.250121 }, "model.layers.19.self_attn.k_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2129920, "bits_per_weight": 3.25, "clipped_fraction": 0.999902, "clip_improvement": 0.25286 }, "model.layers.19.self_attn.o_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999845, "clip_improvement": 0.250766 }, "model.layers.19.self_attn.q_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25559040, "bits_per_weight": 3.25, "clipped_fraction": 0.999839, "clip_improvement": 0.250839 }, "model.layers.19.self_attn.v_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2129920, "bits_per_weight": 3.25, "clipped_fraction": 0.999536, "clip_improvement": 0.253838 }, "model.layers.2.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.2.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.2.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999834, "clip_improvement": 0.250582 }, "model.layers.2.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999805, "clip_improvement": 0.250406 }, "model.layers.2.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999829, "clip_improvement": 0.251606 }, "model.layers.2.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.250175 }, "model.layers.2.mlp.gate_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999961, "clip_improvement": 0.353647 }, "model.layers.2.mlp.up_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999974, "clip_improvement": 0.353523 }, "model.layers.20.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.20.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.20.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999834, "clip_improvement": 0.250579 }, "model.layers.20.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.250433 }, "model.layers.20.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999849, "clip_improvement": 0.251395 }, "model.layers.20.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999859, "clip_improvement": 0.250298 }, "model.layers.20.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999816, "clip_improvement": 0.25006 }, "model.layers.20.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999849, "clip_improvement": 0.250163 }, "model.layers.21.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.21.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.21.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999841, "clip_improvement": 0.250783 }, "model.layers.21.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999898, "clip_improvement": 0.250342 }, "model.layers.21.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999817, "clip_improvement": 0.250441 }, "model.layers.21.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999832, "clip_improvement": 0.25046 }, "model.layers.21.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999841, "clip_improvement": 0.249956 }, "model.layers.21.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999833, "clip_improvement": 0.250088 }, "model.layers.22.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.22.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.22.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999836, "clip_improvement": 0.250619 }, "model.layers.22.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999837, "clip_improvement": 0.250034 }, "model.layers.22.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999837, "clip_improvement": 0.250538 }, "model.layers.22.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999658, "clip_improvement": 0.254971 }, "model.layers.22.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999843, "clip_improvement": 0.250545 }, "model.layers.22.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999843, "clip_improvement": 0.250311 }, "model.layers.23.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999876, "clip_improvement": 0.250967 }, "model.layers.23.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999856, "clip_improvement": 0.250251 }, "model.layers.23.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999836, "clip_improvement": 0.250167 }, "model.layers.23.self_attn.k_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2129920, "bits_per_weight": 3.25, "clipped_fraction": 0.999829, "clip_improvement": 0.253369 }, "model.layers.23.self_attn.o_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999829, "clip_improvement": 0.250832 }, "model.layers.23.self_attn.q_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25559040, "bits_per_weight": 3.25, "clipped_fraction": 0.999829, "clip_improvement": 0.25062 }, "model.layers.23.self_attn.v_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2129920, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.251332 }, "model.layers.24.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.24.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.24.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999895, "clip_improvement": 0.250916 }, "model.layers.24.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999849, "clip_improvement": 0.250412 }, "model.layers.24.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999862, "clip_improvement": 0.250533 }, "model.layers.24.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999838, "clip_improvement": 0.250928 }, "model.layers.24.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999861, "clip_improvement": 0.250359 }, "model.layers.24.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999833, "clip_improvement": 0.250264 }, "model.layers.25.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.25.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.25.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999885, "clip_improvement": 0.250974 }, "model.layers.25.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999809, "clip_improvement": 0.250391 }, "model.layers.25.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999878, "clip_improvement": 0.250444 }, "model.layers.25.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999856, "clip_improvement": 0.250225 }, "model.layers.25.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999843, "clip_improvement": 0.250066 }, "model.layers.25.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999835, "clip_improvement": 0.250287 }, "model.layers.26.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.26.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.26.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999866, "clip_improvement": 0.251135 }, "model.layers.26.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999866, "clip_improvement": 0.250561 }, "model.layers.26.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999858, "clip_improvement": 0.251026 }, "model.layers.26.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999845, "clip_improvement": 0.250635 }, "model.layers.26.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999822, "clip_improvement": 0.250471 }, "model.layers.26.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999866, "clip_improvement": 0.250615 }, "model.layers.27.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999855, "clip_improvement": 0.250789 }, "model.layers.27.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999851, "clip_improvement": 0.25029 }, "model.layers.27.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999861, "clip_improvement": 0.250364 }, "model.layers.27.self_attn.k_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2129920, "bits_per_weight": 3.25, "clipped_fraction": 0.999878, "clip_improvement": 0.254927 }, "model.layers.27.self_attn.o_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999878, "clip_improvement": 0.250476 }, "model.layers.27.self_attn.q_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25559040, "bits_per_weight": 3.25, "clipped_fraction": 0.999829, "clip_improvement": 0.250888 }, "model.layers.27.self_attn.v_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2129920, "bits_per_weight": 3.25, "clipped_fraction": 0.999683, "clip_improvement": 0.25221 }, "model.layers.28.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.28.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.28.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999851, "clip_improvement": 0.251185 }, "model.layers.28.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999821, "clip_improvement": 0.250604 }, "model.layers.28.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999841, "clip_improvement": 0.250859 }, "model.layers.28.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999862, "clip_improvement": 0.250542 }, "model.layers.28.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999841, "clip_improvement": 0.250338 }, "model.layers.28.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999864, "clip_improvement": 0.250489 }, "model.layers.29.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.29.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.29.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999866, "clip_improvement": 0.250794 }, "model.layers.29.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999862, "clip_improvement": 0.250709 }, "model.layers.29.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999886, "clip_improvement": 0.250251 }, "model.layers.29.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999856, "clip_improvement": 0.250275 }, "model.layers.29.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999855, "clip_improvement": 0.250481 }, "model.layers.29.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999838, "clip_improvement": 0.250411 }, "model.layers.3.mlp.down_proj": { "bits": 2, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999974, "clip_improvement": 0.353523 }, "model.layers.3.mlp.gate_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.99999, "clip_improvement": 0.353338 }, "model.layers.3.mlp.up_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999989, "clip_improvement": 0.3536 }, "model.layers.3.self_attn.k_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2129920, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.252481 }, "model.layers.3.self_attn.o_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999845, "clip_improvement": 0.249997 }, "model.layers.3.self_attn.q_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25559040, "bits_per_weight": 3.25, "clipped_fraction": 0.999858, "clip_improvement": 0.250812 }, "model.layers.3.self_attn.v_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2129920, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.250395 }, "model.layers.30.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.30.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.30.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999853, "clip_improvement": 0.251121 }, "model.layers.30.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999874, "clip_improvement": 0.250881 }, "model.layers.30.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999817, "clip_improvement": 0.250916 }, "model.layers.30.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999835, "clip_improvement": 0.250864 }, "model.layers.30.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999829, "clip_improvement": 0.250478 }, "model.layers.30.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999848, "clip_improvement": 0.25042 }, "model.layers.31.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999815, "clip_improvement": 0.250486 }, "model.layers.31.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999862, "clip_improvement": 0.250606 }, "model.layers.31.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.250653 }, "model.layers.31.self_attn.k_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2129920, "bits_per_weight": 3.25, "clipped_fraction": 0.999878, "clip_improvement": 0.252961 }, "model.layers.31.self_attn.o_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999849, "clip_improvement": 0.250563 }, "model.layers.31.self_attn.q_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25559040, "bits_per_weight": 3.25, "clipped_fraction": 0.999872, "clip_improvement": 0.250712 }, "model.layers.31.self_attn.v_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2129920, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.251728 }, "model.layers.32.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.32.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.32.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.99989, "clip_improvement": 0.250942 }, "model.layers.32.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999878, "clip_improvement": 0.250956 }, "model.layers.32.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999845, "clip_improvement": 0.250989 }, "model.layers.32.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999836, "clip_improvement": 0.250786 }, "model.layers.32.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999841, "clip_improvement": 0.250842 }, "model.layers.32.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999849, "clip_improvement": 0.250684 }, "model.layers.33.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.33.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.33.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999888, "clip_improvement": 0.251302 }, "model.layers.33.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999866, "clip_improvement": 0.251082 }, "model.layers.33.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999825, "clip_improvement": 0.250962 }, "model.layers.33.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999859, "clip_improvement": 0.250823 }, "model.layers.33.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999845, "clip_improvement": 0.251086 }, "model.layers.33.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999835, "clip_improvement": 0.250689 }, "model.layers.34.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.34.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.34.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999878, "clip_improvement": 0.251089 }, "model.layers.34.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999805, "clip_improvement": 0.251384 }, "model.layers.34.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.250989 }, "model.layers.34.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999698, "clip_improvement": 0.252964 }, "model.layers.34.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999785, "clip_improvement": 0.252622 }, "model.layers.34.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999839, "clip_improvement": 0.250805 }, "model.layers.35.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.250783 }, "model.layers.35.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999819, "clip_improvement": 0.250965 }, "model.layers.35.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999845, "clip_improvement": 0.250573 }, "model.layers.35.self_attn.k_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2129920, "bits_per_weight": 3.25, "clipped_fraction": 0.99978, "clip_improvement": 0.253111 }, "model.layers.35.self_attn.o_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999862, "clip_improvement": 0.250769 }, "model.layers.35.self_attn.q_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25559040, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.250772 }, "model.layers.35.self_attn.v_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2129920, "bits_per_weight": 3.25, "clipped_fraction": 0.99978, "clip_improvement": 0.253496 }, "model.layers.36.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.36.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.36.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999846, "clip_improvement": 0.250961 }, "model.layers.36.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.99987, "clip_improvement": 0.250413 }, "model.layers.36.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.25111 }, "model.layers.36.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999832, "clip_improvement": 0.250335 }, "model.layers.36.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999845, "clip_improvement": 0.250866 }, "model.layers.36.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999875, "clip_improvement": 0.250406 }, "model.layers.37.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.37.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.37.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999675, "clip_improvement": 0.251486 }, "model.layers.37.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999821, "clip_improvement": 0.250887 }, "model.layers.37.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999813, "clip_improvement": 0.251294 }, "model.layers.37.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999848, "clip_improvement": 0.250149 }, "model.layers.37.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999833, "clip_improvement": 0.250553 }, "model.layers.37.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999835, "clip_improvement": 0.25028 }, "model.layers.38.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.38.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.38.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999839, "clip_improvement": 0.250733 }, "model.layers.38.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999862, "clip_improvement": 0.250613 }, "model.layers.38.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999849, "clip_improvement": 0.250635 }, "model.layers.38.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999841, "clip_improvement": 0.250749 }, "model.layers.38.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999842, "clip_improvement": 0.250539 }, "model.layers.38.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999856, "clip_improvement": 0.250307 }, "model.layers.39.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999881, "clip_improvement": 0.250513 }, "model.layers.39.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999838, "clip_improvement": 0.250093 }, "model.layers.39.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999864, "clip_improvement": 0.250214 }, "model.layers.39.self_attn.k_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2129920, "bits_per_weight": 3.25, "clipped_fraction": 0.999756, "clip_improvement": 0.254395 }, "model.layers.39.self_attn.o_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999841, "clip_improvement": 0.250631 }, "model.layers.39.self_attn.q_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25559040, "bits_per_weight": 3.25, "clipped_fraction": 0.999868, "clip_improvement": 0.250693 }, "model.layers.39.self_attn.v_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2129920, "bits_per_weight": 3.25, "clipped_fraction": 0.999829, "clip_improvement": 0.252474 }, "model.layers.4.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.4.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.4.linear_attn.in_proj_qkv": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 14745600, "bits_per_weight": 2.25, "clipped_fraction": 0.999983, "clip_improvement": 0.353219 }, "model.layers.4.linear_attn.in_proj_z": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999923, "clip_improvement": 0.352992 }, "model.layers.4.linear_attn.out_proj": { "bits": 2, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999939, "clip_improvement": 0.355731 }, "model.layers.4.mlp.down_proj": { "bits": 2, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999957, "clip_improvement": 0.353171 }, "model.layers.4.mlp.gate_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999976, "clip_improvement": 0.353224 }, "model.layers.4.mlp.up_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999994, "clip_improvement": 0.353679 }, "model.layers.40.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.40.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.40.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999841, "clip_improvement": 0.251388 }, "model.layers.40.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999793, "clip_improvement": 0.250878 }, "model.layers.40.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999886, "clip_improvement": 0.251604 }, "model.layers.40.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999848, "clip_improvement": 0.250317 }, "model.layers.40.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.250328 }, "model.layers.40.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999864, "clip_improvement": 0.250118 }, "model.layers.41.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.41.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.41.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999878, "clip_improvement": 0.250711 }, "model.layers.41.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999878, "clip_improvement": 0.250691 }, "model.layers.41.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999845, "clip_improvement": 0.250379 }, "model.layers.41.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999866, "clip_improvement": 0.250512 }, "model.layers.41.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999806, "clip_improvement": 0.250359 }, "model.layers.41.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999875, "clip_improvement": 0.250346 }, "model.layers.42.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.42.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.42.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999792, "clip_improvement": 0.251079 }, "model.layers.42.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999882, "clip_improvement": 0.250732 }, "model.layers.42.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999805, "clip_improvement": 0.251127 }, "model.layers.42.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999835, "clip_improvement": 0.250647 }, "model.layers.42.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999816, "clip_improvement": 0.250717 }, "model.layers.42.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999868, "clip_improvement": 0.250664 }, "model.layers.43.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999843, "clip_improvement": 0.250459 }, "model.layers.43.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999815, "clip_improvement": 0.250305 }, "model.layers.43.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999868, "clip_improvement": 0.250312 }, "model.layers.43.self_attn.k_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2129920, "bits_per_weight": 3.25, "clipped_fraction": 0.999805, "clip_improvement": 0.254231 }, "model.layers.43.self_attn.o_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999882, "clip_improvement": 0.251551 }, "model.layers.43.self_attn.q_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25559040, "bits_per_weight": 3.25, "clipped_fraction": 0.999876, "clip_improvement": 0.251014 }, "model.layers.43.self_attn.v_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2129920, "bits_per_weight": 3.25, "clipped_fraction": 0.999731, "clip_improvement": 0.255129 }, "model.layers.44.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.44.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.44.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999839, "clip_improvement": 0.251042 }, "model.layers.44.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999825, "clip_improvement": 0.25104 }, "model.layers.44.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999829, "clip_improvement": 0.251 }, "model.layers.44.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.250403 }, "model.layers.44.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999828, "clip_improvement": 0.250627 }, "model.layers.44.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999871, "clip_improvement": 0.250432 }, "model.layers.45.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.45.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.45.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999853, "clip_improvement": 0.250857 }, "model.layers.45.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999821, "clip_improvement": 0.250449 }, "model.layers.45.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.99987, "clip_improvement": 0.25036 }, "model.layers.45.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999836, "clip_improvement": 0.250448 }, "model.layers.45.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999819, "clip_improvement": 0.250478 }, "model.layers.45.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999872, "clip_improvement": 0.25047 }, "model.layers.46.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.46.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.46.linear_attn.in_proj_qkv": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 14745600, "bits_per_weight": 2.25, "clipped_fraction": 0.999744, "clip_improvement": 0.354854 }, "model.layers.46.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999849, "clip_improvement": 0.25129 }, "model.layers.46.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999837, "clip_improvement": 0.250893 }, "model.layers.46.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999831, "clip_improvement": 0.25042 }, "model.layers.46.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.250671 }, "model.layers.46.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999831, "clip_improvement": 0.250489 }, "model.layers.47.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999833, "clip_improvement": 0.250788 }, "model.layers.47.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999826, "clip_improvement": 0.250689 }, "model.layers.47.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999865, "clip_improvement": 0.250492 }, "model.layers.47.self_attn.k_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1474560, "bits_per_weight": 2.25, "clipped_fraction": 0.999048, "clip_improvement": 0.353625 }, "model.layers.47.self_attn.o_proj": { "bits": 2, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999923, "clip_improvement": 0.354173 }, "model.layers.47.self_attn.q_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 17694720, "bits_per_weight": 2.25, "clipped_fraction": 0.999906, "clip_improvement": 0.354033 }, "model.layers.47.self_attn.v_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2129920, "bits_per_weight": 3.25, "clipped_fraction": 0.999829, "clip_improvement": 0.251878 }, "model.layers.48.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.48.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.48.linear_attn.in_proj_qkv": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 14745600, "bits_per_weight": 2.25, "clipped_fraction": 0.999922, "clip_improvement": 0.355062 }, "model.layers.48.linear_attn.in_proj_z": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999882, "clip_improvement": 0.353672 }, "model.layers.48.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999845, "clip_improvement": 0.251565 }, "model.layers.48.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999861, "clip_improvement": 0.25088 }, "model.layers.48.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999812, "clip_improvement": 0.250925 }, "model.layers.48.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999846, "clip_improvement": 0.250546 }, "model.layers.49.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.49.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.49.linear_attn.in_proj_qkv": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 14745600, "bits_per_weight": 2.25, "clipped_fraction": 0.999905, "clip_improvement": 0.354578 }, "model.layers.49.linear_attn.in_proj_z": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999963, "clip_improvement": 0.353659 }, "model.layers.49.linear_attn.out_proj": { "bits": 2, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999902, "clip_improvement": 0.35385 }, "model.layers.49.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999845, "clip_improvement": 0.250922 }, "model.layers.49.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999759, "clip_improvement": 0.251713 }, "model.layers.49.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999856, "clip_improvement": 0.250509 }, "model.layers.5.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.5.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.5.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999841, "clip_improvement": 0.25045 }, "model.layers.5.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999854, "clip_improvement": 0.250406 }, "model.layers.5.linear_attn.out_proj": { "bits": 2, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999935, "clip_improvement": 0.353915 }, "model.layers.5.mlp.down_proj": { "bits": 2, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999958, "clip_improvement": 0.353008 }, "model.layers.5.mlp.gate_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999968, "clip_improvement": 0.353494 }, "model.layers.5.mlp.up_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999994, "clip_improvement": 0.353572 }, "model.layers.50.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.50.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.50.linear_attn.in_proj_qkv": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 14745600, "bits_per_weight": 2.25, "clipped_fraction": 0.999585, "clip_improvement": 0.354729 }, "model.layers.50.linear_attn.in_proj_z": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999927, "clip_improvement": 0.354149 }, "model.layers.50.linear_attn.out_proj": { "bits": 2, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.99974, "clip_improvement": 0.355047 }, "model.layers.50.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999796, "clip_improvement": 0.252372 }, "model.layers.50.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999812, "clip_improvement": 0.251848 }, "model.layers.50.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999845, "clip_improvement": 0.25089 }, "model.layers.51.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999829, "clip_improvement": 0.251376 }, "model.layers.51.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999836, "clip_improvement": 0.25114 }, "model.layers.51.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999845, "clip_improvement": 0.250395 }, "model.layers.51.self_attn.k_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1474560, "bits_per_weight": 2.25, "clipped_fraction": 0.998633, "clip_improvement": 0.35126 }, "model.layers.51.self_attn.o_proj": { "bits": 2, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999833, "clip_improvement": 0.355382 }, "model.layers.51.self_attn.q_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 17694720, "bits_per_weight": 2.25, "clipped_fraction": 0.99999, "clip_improvement": 0.354918 }, "model.layers.51.self_attn.v_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1474560, "bits_per_weight": 2.25, "clipped_fraction": 0.994458, "clip_improvement": 0.357812 }, "model.layers.52.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.52.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.52.linear_attn.in_proj_qkv": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 14745600, "bits_per_weight": 2.25, "clipped_fraction": 0.999807, "clip_improvement": 0.352872 }, "model.layers.52.linear_attn.in_proj_z": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999927, "clip_improvement": 0.354082 }, "model.layers.52.linear_attn.out_proj": { "bits": 2, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999675, "clip_improvement": 0.356662 }, "model.layers.52.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999848, "clip_improvement": 0.250889 }, "model.layers.52.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999819, "clip_improvement": 0.251123 }, "model.layers.52.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999868, "clip_improvement": 0.250569 }, "model.layers.53.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.53.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.53.linear_attn.in_proj_qkv": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 14745600, "bits_per_weight": 2.25, "clipped_fraction": 0.999705, "clip_improvement": 0.35335 }, "model.layers.53.linear_attn.in_proj_z": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999902, "clip_improvement": 0.353271 }, "model.layers.53.linear_attn.out_proj": { "bits": 2, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.99967, "clip_improvement": 0.355639 }, "model.layers.53.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999828, "clip_improvement": 0.251118 }, "model.layers.53.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999829, "clip_improvement": 0.250976 }, "model.layers.53.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999845, "clip_improvement": 0.250399 }, "model.layers.54.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.54.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.54.linear_attn.in_proj_qkv": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 14745600, "bits_per_weight": 2.25, "clipped_fraction": 0.999565, "clip_improvement": 0.353618 }, "model.layers.54.linear_attn.in_proj_z": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999976, "clip_improvement": 0.353476 }, "model.layers.54.linear_attn.out_proj": { "bits": 2, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999658, "clip_improvement": 0.357367 }, "model.layers.54.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999819, "clip_improvement": 0.25104 }, "model.layers.54.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999855, "clip_improvement": 0.250931 }, "model.layers.54.mlp.up_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999917, "clip_improvement": 0.354269 }, "model.layers.55.mlp.down_proj": { "bits": 2, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999947, "clip_improvement": 0.3541 }, "model.layers.55.mlp.gate_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999937, "clip_improvement": 0.353635 }, "model.layers.55.mlp.up_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999977, "clip_improvement": 0.353981 }, "model.layers.55.self_attn.k_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1474560, "bits_per_weight": 2.25, "clipped_fraction": 0.997241, "clip_improvement": 0.347822 }, "model.layers.55.self_attn.o_proj": { "bits": 2, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999825, "clip_improvement": 0.354671 }, "model.layers.55.self_attn.q_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 17694720, "bits_per_weight": 2.25, "clipped_fraction": 0.999939, "clip_improvement": 0.353314 }, "model.layers.55.self_attn.v_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1474560, "bits_per_weight": 2.25, "clipped_fraction": 0.997168, "clip_improvement": 0.353878 }, "model.layers.56.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.56.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.56.linear_attn.in_proj_qkv": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 14745600, "bits_per_weight": 2.25, "clipped_fraction": 0.999871, "clip_improvement": 0.352875 }, "model.layers.56.linear_attn.in_proj_z": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999972, "clip_improvement": 0.353063 }, "model.layers.56.linear_attn.out_proj": { "bits": 2, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999813, "clip_improvement": 0.358141 }, "model.layers.56.mlp.down_proj": { "bits": 2, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999964, "clip_improvement": 0.353904 }, "model.layers.56.mlp.gate_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.99992, "clip_improvement": 0.353673 }, "model.layers.56.mlp.up_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999977, "clip_improvement": 0.353969 }, "model.layers.57.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.57.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.57.linear_attn.in_proj_qkv": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 14745600, "bits_per_weight": 2.25, "clipped_fraction": 0.999739, "clip_improvement": 0.352639 }, "model.layers.57.linear_attn.in_proj_z": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999955, "clip_improvement": 0.352948 }, "model.layers.57.linear_attn.out_proj": { "bits": 2, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999605, "clip_improvement": 0.355793 }, "model.layers.57.mlp.down_proj": { "bits": 2, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999984, "clip_improvement": 0.353749 }, "model.layers.57.mlp.gate_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999925, "clip_improvement": 0.353474 }, "model.layers.57.mlp.up_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999974, "clip_improvement": 0.353772 }, "model.layers.58.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.58.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.58.linear_attn.in_proj_qkv": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 14745600, "bits_per_weight": 2.25, "clipped_fraction": 0.999749, "clip_improvement": 0.352913 }, "model.layers.58.linear_attn.in_proj_z": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999963, "clip_improvement": 0.353247 }, "model.layers.58.linear_attn.out_proj": { "bits": 2, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999837, "clip_improvement": 0.357009 }, "model.layers.58.mlp.down_proj": { "bits": 2, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999977, "clip_improvement": 0.354107 }, "model.layers.58.mlp.gate_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999854, "clip_improvement": 0.353698 }, "model.layers.58.mlp.up_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999968, "clip_improvement": 0.353992 }, "model.layers.59.mlp.down_proj": { "bits": 2, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999958, "clip_improvement": 0.355231 }, "model.layers.59.mlp.gate_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999895, "clip_improvement": 0.35364 }, "model.layers.59.mlp.up_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999945, "clip_improvement": 0.354107 }, "model.layers.59.self_attn.k_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1474560, "bits_per_weight": 2.25, "clipped_fraction": 0.996924, "clip_improvement": 0.347415 }, "model.layers.59.self_attn.o_proj": { "bits": 2, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999862, "clip_improvement": 0.355754 }, "model.layers.59.self_attn.q_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 17694720, "bits_per_weight": 2.25, "clipped_fraction": 0.999707, "clip_improvement": 0.352399 }, "model.layers.59.self_attn.v_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1474560, "bits_per_weight": 2.25, "clipped_fraction": 0.997217, "clip_improvement": 0.358638 }, "model.layers.6.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.6.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.6.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999844, "clip_improvement": 0.250666 }, "model.layers.6.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999878, "clip_improvement": 0.250162 }, "model.layers.6.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999849, "clip_improvement": 0.251858 }, "model.layers.6.mlp.down_proj": { "bits": 2, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999269, "clip_improvement": 0.358459 }, "model.layers.6.mlp.gate_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999792, "clip_improvement": 0.250542 }, "model.layers.6.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999835, "clip_improvement": 0.250559 }, "model.layers.60.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.60.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.60.linear_attn.in_proj_qkv": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 14745600, "bits_per_weight": 2.25, "clipped_fraction": 0.99947, "clip_improvement": 0.352356 }, "model.layers.60.linear_attn.in_proj_z": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999866, "clip_improvement": 0.352588 }, "model.layers.60.linear_attn.out_proj": { "bits": 2, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999618, "clip_improvement": 0.357714 }, "model.layers.60.mlp.down_proj": { "bits": 2, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999978, "clip_improvement": 0.354047 }, "model.layers.60.mlp.gate_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999892, "clip_improvement": 0.353722 }, "model.layers.60.mlp.up_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999967, "clip_improvement": 0.353972 }, "model.layers.61.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.61.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.61.linear_attn.in_proj_qkv": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 14745600, "bits_per_weight": 2.25, "clipped_fraction": 0.999663, "clip_improvement": 0.352677 }, "model.layers.61.linear_attn.in_proj_z": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999858, "clip_improvement": 0.352217 }, "model.layers.61.linear_attn.out_proj": { "bits": 2, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999776, "clip_improvement": 0.354437 }, "model.layers.61.mlp.down_proj": { "bits": 2, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999966, "clip_improvement": 0.354029 }, "model.layers.61.mlp.gate_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999869, "clip_improvement": 0.353555 }, "model.layers.61.mlp.up_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999957, "clip_improvement": 0.354352 }, "model.layers.62.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.62.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.62.linear_attn.in_proj_qkv": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 14745600, "bits_per_weight": 2.25, "clipped_fraction": 0.99897, "clip_improvement": 0.3539 }, "model.layers.62.linear_attn.in_proj_z": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999548, "clip_improvement": 0.352459 }, "model.layers.62.linear_attn.out_proj": { "bits": 2, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999544, "clip_improvement": 0.358339 }, "model.layers.62.mlp.down_proj": { "bits": 2, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999865, "clip_improvement": 0.355188 }, "model.layers.62.mlp.gate_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999869, "clip_improvement": 0.35315 }, "model.layers.62.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999818, "clip_improvement": 0.250395 }, "model.layers.63.mlp.down_proj": { "bits": 3, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999875, "clip_improvement": 0.252275 }, "model.layers.63.mlp.gate_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999799, "clip_improvement": 0.353308 }, "model.layers.63.mlp.up_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 36208640, "bits_per_weight": 3.25, "clipped_fraction": 0.999833, "clip_improvement": 0.250269 }, "model.layers.63.self_attn.k_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2129920, "bits_per_weight": 3.25, "clipped_fraction": 0.999707, "clip_improvement": 0.256872 }, "model.layers.63.self_attn.o_proj": { "bits": 2, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 8847360, "bits_per_weight": 2.25, "clipped_fraction": 0.999581, "clip_improvement": 0.358974 }, "model.layers.63.self_attn.q_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 17694720, "bits_per_weight": 2.25, "clipped_fraction": 0.999632, "clip_improvement": 0.352072 }, "model.layers.63.self_attn.v_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1474560, "bits_per_weight": 2.25, "clipped_fraction": 0.995996, "clip_improvement": 0.358435 }, "model.layers.7.mlp.down_proj": { "bits": 2, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999981, "clip_improvement": 0.353291 }, "model.layers.7.mlp.gate_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999978, "clip_improvement": 0.353568 }, "model.layers.7.mlp.up_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999997, "clip_improvement": 0.353725 }, "model.layers.7.self_attn.k_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2129920, "bits_per_weight": 3.25, "clipped_fraction": 0.999902, "clip_improvement": 0.252616 }, "model.layers.7.self_attn.o_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999862, "clip_improvement": 0.250596 }, "model.layers.7.self_attn.q_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 12288, 5120 ], "num_rows": 12288, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25559040, "bits_per_weight": 3.25, "clipped_fraction": 0.999835, "clip_improvement": 0.250956 }, "model.layers.7.self_attn.v_proj": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 1024, 5120 ], "num_rows": 1024, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 2129920, "bits_per_weight": 3.25, "clipped_fraction": 0.99978, "clip_improvement": 0.251199 }, "model.layers.8.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.8.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.8.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999853, "clip_improvement": 0.25062 }, "model.layers.8.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999866, "clip_improvement": 0.250399 }, "model.layers.8.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999874, "clip_improvement": 0.25138 }, "model.layers.8.mlp.down_proj": { "bits": 2, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999991, "clip_improvement": 0.353481 }, "model.layers.8.mlp.gate_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999983, "clip_improvement": 0.353445 }, "model.layers.8.mlp.up_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999997, "clip_improvement": 0.353717 }, "model.layers.9.linear_attn.in_proj_a": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.9.linear_attn.in_proj_b": { "bits": 8, "group_size": 128, "in_features": 5120, "logical_shape": [ 48, 5120 ], "num_rows": 48, "num_groups": 40, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 253440, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.9.linear_attn.in_proj_qkv": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 10240, 5120 ], "num_rows": 10240, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 21299200, "bits_per_weight": 3.25, "clipped_fraction": 0.999849, "clip_improvement": 0.250644 }, "model.layers.9.linear_attn.in_proj_z": { "bits": 3, "group_size": 128, "in_features": 5120, "logical_shape": [ 6144, 5120 ], "num_rows": 6144, "num_groups": 40, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999833, "clip_improvement": 0.250352 }, "model.layers.9.linear_attn.out_proj": { "bits": 3, "group_size": 128, "in_features": 6144, "logical_shape": [ 5120, 6144 ], "num_rows": 5120, "num_groups": 48, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 12779520, "bits_per_weight": 3.25, "clipped_fraction": 0.999849, "clip_improvement": 0.250981 }, "model.layers.9.mlp.down_proj": { "bits": 2, "group_size": 128, "in_features": 17408, "logical_shape": [ 5120, 17408 ], "num_rows": 5120, "num_groups": 136, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999977, "clip_improvement": 0.353306 }, "model.layers.9.mlp.gate_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999951, "clip_improvement": 0.35341 }, "model.layers.9.mlp.up_proj": { "bits": 2, "group_size": 128, "in_features": 5120, "logical_shape": [ 17408, 5120 ], "num_rows": 17408, "num_groups": 40, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 25067520, "bits_per_weight": 2.25, "clipped_fraction": 0.999999, "clip_improvement": 0.353645 } } }