manifest.json

{"model.layers.0.self_attn.q_proj.bits": {"offset": 0, "shape": [1296, 24], "dtype": "u32"}, "model.layers.0.self_attn.q_proj.su": {"offset": 124416, "shape": [576], "dtype": "f16"}, "model.layers.0.self_attn.q_proj.sv": {"offset": 125568, "shape": [576], "dtype": "f16"}, "model.layers.0.self_attn.q_proj.isc": {"offset": 126720, "shape": [576], "dtype": "f16"}, "model.layers.0.self_attn.q_proj.osc": {"offset": 127872, "shape": [576], "dtype": "f16"}, "model.layers.0.self_attn.q_proj.scale": {"offset": 129024, "shape": [1], "dtype": "f32"}, "model.layers.0.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.0.self_attn.k_proj.bits": {"offset": 129028, "shape": [432, 24], "dtype": "u32"}, "model.layers.0.self_attn.k_proj.su": {"offset": 170500, "shape": [576], "dtype": "f16"}, "model.layers.0.self_attn.k_proj.sv": {"offset": 171652, "shape": [192], "dtype": "f16"}, "model.layers.0.self_attn.k_proj.isc": {"offset": 172036, "shape": [576], "dtype": "f16"}, "model.layers.0.self_attn.k_proj.osc": {"offset": 173188, "shape": [192], "dtype": "f16"}, "model.layers.0.self_attn.k_proj.scale": {"offset": 173572, "shape": [1], "dtype": "f32"}, "model.layers.0.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.0.self_attn.v_proj.bits": {"offset": 173576, "shape": [432, 24], "dtype": "u32"}, "model.layers.0.self_attn.v_proj.su": {"offset": 215048, "shape": [576], "dtype": "f16"}, "model.layers.0.self_attn.v_proj.sv": {"offset": 216200, "shape": [192], "dtype": "f16"}, "model.layers.0.self_attn.v_proj.isc": {"offset": 216584, "shape": [576], "dtype": "f16"}, "model.layers.0.self_attn.v_proj.osc": {"offset": 217736, "shape": [192], "dtype": "f16"}, "model.layers.0.self_attn.v_proj.scale": {"offset": 218120, "shape": [1], "dtype": "f32"}, "model.layers.0.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.0.self_attn.o_proj.bits": {"offset": 218124, "shape": [1296, 24], "dtype": "u32"}, "model.layers.0.self_attn.o_proj.su": {"offset": 342540, "shape": [576], "dtype": "f16"}, "model.layers.0.self_attn.o_proj.sv": {"offset": 343692, "shape": [576], "dtype": "f16"}, "model.layers.0.self_attn.o_proj.isc": {"offset": 344844, "shape": [576], "dtype": "f16"}, "model.layers.0.self_attn.o_proj.osc": {"offset": 345996, "shape": [576], "dtype": "f16"}, "model.layers.0.self_attn.o_proj.scale": {"offset": 347148, "shape": [1], "dtype": "f32"}, "model.layers.0.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.0.mlp.gate_proj.bits": {"offset": 347152, "shape": [3456, 24], "dtype": "u32"}, "model.layers.0.mlp.gate_proj.su": {"offset": 678928, "shape": [576], "dtype": "f16"}, "model.layers.0.mlp.gate_proj.sv": {"offset": 680080, "shape": [1536], "dtype": "f16"}, "model.layers.0.mlp.gate_proj.isc": {"offset": 683152, "shape": [576], "dtype": "f16"}, "model.layers.0.mlp.gate_proj.osc": {"offset": 684304, "shape": [1536], "dtype": "f16"}, "model.layers.0.mlp.gate_proj.scale": {"offset": 687376, "shape": [1], "dtype": "f32"}, "model.layers.0.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.0.mlp.up_proj.bits": {"offset": 687380, "shape": [3456, 24], "dtype": "u32"}, "model.layers.0.mlp.up_proj.su": {"offset": 1019156, "shape": [576], "dtype": "f16"}, "model.layers.0.mlp.up_proj.sv": {"offset": 1020308, "shape": [1536], "dtype": "f16"}, "model.layers.0.mlp.up_proj.isc": {"offset": 1023380, "shape": [576], "dtype": "f16"}, "model.layers.0.mlp.up_proj.osc": {"offset": 1024532, "shape": [1536], "dtype": "f16"}, "model.layers.0.mlp.up_proj.scale": {"offset": 1027604, "shape": [1], "dtype": "f32"}, "model.layers.0.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.0.mlp.down_proj.bits": {"offset": 1027608, "shape": [3456, 24], "dtype": "u32"}, "model.layers.0.mlp.down_proj.su": {"offset": 1359384, "shape": [1536], "dtype": "f16"}, "model.layers.0.mlp.down_proj.sv": {"offset": 1362456, "shape": [576], "dtype": "f16"}, "model.layers.0.mlp.down_proj.isc": {"offset": 1363608, "shape": [1536], "dtype": "f16"}, "model.layers.0.mlp.down_proj.osc": {"offset": 1366680, "shape": [576], "dtype": "f16"}, "model.layers.0.mlp.down_proj.scale": {"offset": 1367832, "shape": [1], "dtype": "f32"}, "model.layers.0.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.1.self_attn.q_proj.bits": {"offset": 1367836, "shape": [1296, 24], "dtype": "u32"}, "model.layers.1.self_attn.q_proj.su": {"offset": 1492252, "shape": [576], "dtype": "f16"}, "model.layers.1.self_attn.q_proj.sv": {"offset": 1493404, "shape": [576], "dtype": "f16"}, "model.layers.1.self_attn.q_proj.isc": {"offset": 1494556, "shape": [576], "dtype": "f16"}, "model.layers.1.self_attn.q_proj.osc": {"offset": 1495708, "shape": [576], "dtype": "f16"}, "model.layers.1.self_attn.q_proj.scale": {"offset": 1496860, "shape": [1], "dtype": "f32"}, "model.layers.1.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.1.self_attn.k_proj.bits": {"offset": 1496864, "shape": [432, 24], "dtype": "u32"}, "model.layers.1.self_attn.k_proj.su": {"offset": 1538336, "shape": [576], "dtype": "f16"}, "model.layers.1.self_attn.k_proj.sv": {"offset": 1539488, "shape": [192], "dtype": "f16"}, "model.layers.1.self_attn.k_proj.isc": {"offset": 1539872, "shape": [576], "dtype": "f16"}, "model.layers.1.self_attn.k_proj.osc": {"offset": 1541024, "shape": [192], "dtype": "f16"}, "model.layers.1.self_attn.k_proj.scale": {"offset": 1541408, "shape": [1], "dtype": "f32"}, "model.layers.1.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.1.self_attn.v_proj.bits": {"offset": 1541412, "shape": [432, 24], "dtype": "u32"}, "model.layers.1.self_attn.v_proj.su": {"offset": 1582884, "shape": [576], "dtype": "f16"}, "model.layers.1.self_attn.v_proj.sv": {"offset": 1584036, "shape": [192], "dtype": "f16"}, "model.layers.1.self_attn.v_proj.isc": {"offset": 1584420, "shape": [576], "dtype": "f16"}, "model.layers.1.self_attn.v_proj.osc": {"offset": 1585572, "shape": [192], "dtype": "f16"}, "model.layers.1.self_attn.v_proj.scale": {"offset": 1585956, "shape": [1], "dtype": "f32"}, "model.layers.1.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.1.self_attn.o_proj.bits": {"offset": 1585960, "shape": [1296, 24], "dtype": "u32"}, "model.layers.1.self_attn.o_proj.su": {"offset": 1710376, "shape": [576], "dtype": "f16"}, "model.layers.1.self_attn.o_proj.sv": {"offset": 1711528, "shape": [576], "dtype": "f16"}, "model.layers.1.self_attn.o_proj.isc": {"offset": 1712680, "shape": [576], "dtype": "f16"}, "model.layers.1.self_attn.o_proj.osc": {"offset": 1713832, "shape": [576], "dtype": "f16"}, "model.layers.1.self_attn.o_proj.scale": {"offset": 1714984, "shape": [1], "dtype": "f32"}, "model.layers.1.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.1.mlp.gate_proj.bits": {"offset": 1714988, "shape": [3456, 24], "dtype": "u32"}, "model.layers.1.mlp.gate_proj.su": {"offset": 2046764, "shape": [576], "dtype": "f16"}, "model.layers.1.mlp.gate_proj.sv": {"offset": 2047916, "shape": [1536], "dtype": "f16"}, "model.layers.1.mlp.gate_proj.isc": {"offset": 2050988, "shape": [576], "dtype": "f16"}, "model.layers.1.mlp.gate_proj.osc": {"offset": 2052140, "shape": [1536], "dtype": "f16"}, "model.layers.1.mlp.gate_proj.scale": {"offset": 2055212, "shape": [1], "dtype": "f32"}, "model.layers.1.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.1.mlp.up_proj.bits": {"offset": 2055216, "shape": [3456, 24], "dtype": "u32"}, "model.layers.1.mlp.up_proj.su": {"offset": 2386992, "shape": [576], "dtype": "f16"}, "model.layers.1.mlp.up_proj.sv": {"offset": 2388144, "shape": [1536], "dtype": "f16"}, "model.layers.1.mlp.up_proj.isc": {"offset": 2391216, "shape": [576], "dtype": "f16"}, "model.layers.1.mlp.up_proj.osc": {"offset": 2392368, "shape": [1536], "dtype": "f16"}, "model.layers.1.mlp.up_proj.scale": {"offset": 2395440, "shape": [1], "dtype": "f32"}, "model.layers.1.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.1.mlp.down_proj.bits": {"offset": 2395444, "shape": [3456, 24], "dtype": "u32"}, "model.layers.1.mlp.down_proj.su": {"offset": 2727220, "shape": [1536], "dtype": "f16"}, "model.layers.1.mlp.down_proj.sv": {"offset": 2730292, "shape": [576], "dtype": "f16"}, "model.layers.1.mlp.down_proj.isc": {"offset": 2731444, "shape": [1536], "dtype": "f16"}, "model.layers.1.mlp.down_proj.osc": {"offset": 2734516, "shape": [576], "dtype": "f16"}, "model.layers.1.mlp.down_proj.scale": {"offset": 2735668, "shape": [1], "dtype": "f32"}, "model.layers.1.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.2.self_attn.q_proj.bits": {"offset": 2735672, "shape": [1296, 24], "dtype": "u32"}, "model.layers.2.self_attn.q_proj.su": {"offset": 2860088, "shape": [576], "dtype": "f16"}, "model.layers.2.self_attn.q_proj.sv": {"offset": 2861240, "shape": [576], "dtype": "f16"}, "model.layers.2.self_attn.q_proj.isc": {"offset": 2862392, "shape": [576], "dtype": "f16"}, "model.layers.2.self_attn.q_proj.osc": {"offset": 2863544, "shape": [576], "dtype": "f16"}, "model.layers.2.self_attn.q_proj.scale": {"offset": 2864696, "shape": [1], "dtype": "f32"}, "model.layers.2.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.2.self_attn.k_proj.bits": {"offset": 2864700, "shape": [432, 24], "dtype": "u32"}, "model.layers.2.self_attn.k_proj.su": {"offset": 2906172, "shape": [576], "dtype": "f16"}, "model.layers.2.self_attn.k_proj.sv": {"offset": 2907324, "shape": [192], "dtype": "f16"}, "model.layers.2.self_attn.k_proj.isc": {"offset": 2907708, "shape": [576], "dtype": "f16"}, "model.layers.2.self_attn.k_proj.osc": {"offset": 2908860, "shape": [192], "dtype": "f16"}, "model.layers.2.self_attn.k_proj.scale": {"offset": 2909244, "shape": [1], "dtype": "f32"}, "model.layers.2.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.2.self_attn.v_proj.bits": {"offset": 2909248, "shape": [432, 24], "dtype": "u32"}, "model.layers.2.self_attn.v_proj.su": {"offset": 2950720, "shape": [576], "dtype": "f16"}, "model.layers.2.self_attn.v_proj.sv": {"offset": 2951872, "shape": [192], "dtype": "f16"}, "model.layers.2.self_attn.v_proj.isc": {"offset": 2952256, "shape": [576], "dtype": "f16"}, "model.layers.2.self_attn.v_proj.osc": {"offset": 2953408, "shape": [192], "dtype": "f16"}, "model.layers.2.self_attn.v_proj.scale": {"offset": 2953792, "shape": [1], "dtype": "f32"}, "model.layers.2.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.2.self_attn.o_proj.bits": {"offset": 2953796, "shape": [1296, 24], "dtype": "u32"}, "model.layers.2.self_attn.o_proj.su": {"offset": 3078212, "shape": [576], "dtype": "f16"}, "model.layers.2.self_attn.o_proj.sv": {"offset": 3079364, "shape": [576], "dtype": "f16"}, "model.layers.2.self_attn.o_proj.isc": {"offset": 3080516, "shape": [576], "dtype": "f16"}, "model.layers.2.self_attn.o_proj.osc": {"offset": 3081668, "shape": [576], "dtype": "f16"}, "model.layers.2.self_attn.o_proj.scale": {"offset": 3082820, "shape": [1], "dtype": "f32"}, "model.layers.2.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.2.mlp.gate_proj.bits": {"offset": 3082824, "shape": [3456, 24], "dtype": "u32"}, "model.layers.2.mlp.gate_proj.su": {"offset": 3414600, "shape": [576], "dtype": "f16"}, "model.layers.2.mlp.gate_proj.sv": {"offset": 3415752, "shape": [1536], "dtype": "f16"}, "model.layers.2.mlp.gate_proj.isc": {"offset": 3418824, "shape": [576], "dtype": "f16"}, "model.layers.2.mlp.gate_proj.osc": {"offset": 3419976, "shape": [1536], "dtype": "f16"}, "model.layers.2.mlp.gate_proj.scale": {"offset": 3423048, "shape": [1], "dtype": "f32"}, "model.layers.2.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.2.mlp.up_proj.bits": {"offset": 3423052, "shape": [3456, 24], "dtype": "u32"}, "model.layers.2.mlp.up_proj.su": {"offset": 3754828, "shape": [576], "dtype": "f16"}, "model.layers.2.mlp.up_proj.sv": {"offset": 3755980, "shape": [1536], "dtype": "f16"}, "model.layers.2.mlp.up_proj.isc": {"offset": 3759052, "shape": [576], "dtype": "f16"}, "model.layers.2.mlp.up_proj.osc": {"offset": 3760204, "shape": [1536], "dtype": "f16"}, "model.layers.2.mlp.up_proj.scale": {"offset": 3763276, "shape": [1], "dtype": "f32"}, "model.layers.2.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.2.mlp.down_proj.bits": {"offset": 3763280, "shape": [3456, 24], "dtype": "u32"}, "model.layers.2.mlp.down_proj.su": {"offset": 4095056, "shape": [1536], "dtype": "f16"}, "model.layers.2.mlp.down_proj.sv": {"offset": 4098128, "shape": [576], "dtype": "f16"}, "model.layers.2.mlp.down_proj.isc": {"offset": 4099280, "shape": [1536], "dtype": "f16"}, "model.layers.2.mlp.down_proj.osc": {"offset": 4102352, "shape": [576], "dtype": "f16"}, "model.layers.2.mlp.down_proj.scale": {"offset": 4103504, "shape": [1], "dtype": "f32"}, "model.layers.2.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.3.self_attn.q_proj.bits": {"offset": 4103508, "shape": [1296, 24], "dtype": "u32"}, "model.layers.3.self_attn.q_proj.su": {"offset": 4227924, "shape": [576], "dtype": "f16"}, "model.layers.3.self_attn.q_proj.sv": {"offset": 4229076, "shape": [576], "dtype": "f16"}, "model.layers.3.self_attn.q_proj.isc": {"offset": 4230228, "shape": [576], "dtype": "f16"}, "model.layers.3.self_attn.q_proj.osc": {"offset": 4231380, "shape": [576], "dtype": "f16"}, "model.layers.3.self_attn.q_proj.scale": {"offset": 4232532, "shape": [1], "dtype": "f32"}, "model.layers.3.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.3.self_attn.k_proj.bits": {"offset": 4232536, "shape": [432, 24], "dtype": "u32"}, "model.layers.3.self_attn.k_proj.su": {"offset": 4274008, "shape": [576], "dtype": "f16"}, "model.layers.3.self_attn.k_proj.sv": {"offset": 4275160, "shape": [192], "dtype": "f16"}, "model.layers.3.self_attn.k_proj.isc": {"offset": 4275544, "shape": [576], "dtype": "f16"}, "model.layers.3.self_attn.k_proj.osc": {"offset": 4276696, "shape": [192], "dtype": "f16"}, "model.layers.3.self_attn.k_proj.scale": {"offset": 4277080, "shape": [1], "dtype": "f32"}, "model.layers.3.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.3.self_attn.v_proj.bits": {"offset": 4277084, "shape": [432, 24], "dtype": "u32"}, "model.layers.3.self_attn.v_proj.su": {"offset": 4318556, "shape": [576], "dtype": "f16"}, "model.layers.3.self_attn.v_proj.sv": {"offset": 4319708, "shape": [192], "dtype": "f16"}, "model.layers.3.self_attn.v_proj.isc": {"offset": 4320092, "shape": [576], "dtype": "f16"}, "model.layers.3.self_attn.v_proj.osc": {"offset": 4321244, "shape": [192], "dtype": "f16"}, "model.layers.3.self_attn.v_proj.scale": {"offset": 4321628, "shape": [1], "dtype": "f32"}, "model.layers.3.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.3.self_attn.o_proj.bits": {"offset": 4321632, "shape": [1296, 24], "dtype": "u32"}, "model.layers.3.self_attn.o_proj.su": {"offset": 4446048, "shape": [576], "dtype": "f16"}, "model.layers.3.self_attn.o_proj.sv": {"offset": 4447200, "shape": [576], "dtype": "f16"}, "model.layers.3.self_attn.o_proj.isc": {"offset": 4448352, "shape": [576], "dtype": "f16"}, "model.layers.3.self_attn.o_proj.osc": {"offset": 4449504, "shape": [576], "dtype": "f16"}, "model.layers.3.self_attn.o_proj.scale": {"offset": 4450656, "shape": [1], "dtype": "f32"}, "model.layers.3.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.3.mlp.gate_proj.bits": {"offset": 4450660, "shape": [3456, 24], "dtype": "u32"}, "model.layers.3.mlp.gate_proj.su": {"offset": 4782436, "shape": [576], "dtype": "f16"}, "model.layers.3.mlp.gate_proj.sv": {"offset": 4783588, "shape": [1536], "dtype": "f16"}, "model.layers.3.mlp.gate_proj.isc": {"offset": 4786660, "shape": [576], "dtype": "f16"}, "model.layers.3.mlp.gate_proj.osc": {"offset": 4787812, "shape": [1536], "dtype": "f16"}, "model.layers.3.mlp.gate_proj.scale": {"offset": 4790884, "shape": [1], "dtype": "f32"}, "model.layers.3.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.3.mlp.up_proj.bits": {"offset": 4790888, "shape": [3456, 24], "dtype": "u32"}, "model.layers.3.mlp.up_proj.su": {"offset": 5122664, "shape": [576], "dtype": "f16"}, "model.layers.3.mlp.up_proj.sv": {"offset": 5123816, "shape": [1536], "dtype": "f16"}, "model.layers.3.mlp.up_proj.isc": {"offset": 5126888, "shape": [576], "dtype": "f16"}, "model.layers.3.mlp.up_proj.osc": {"offset": 5128040, "shape": [1536], "dtype": "f16"}, "model.layers.3.mlp.up_proj.scale": {"offset": 5131112, "shape": [1], "dtype": "f32"}, "model.layers.3.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.3.mlp.down_proj.bits": {"offset": 5131116, "shape": [3456, 24], "dtype": "u32"}, "model.layers.3.mlp.down_proj.su": {"offset": 5462892, "shape": [1536], "dtype": "f16"}, "model.layers.3.mlp.down_proj.sv": {"offset": 5465964, "shape": [576], "dtype": "f16"}, "model.layers.3.mlp.down_proj.isc": {"offset": 5467116, "shape": [1536], "dtype": "f16"}, "model.layers.3.mlp.down_proj.osc": {"offset": 5470188, "shape": [576], "dtype": "f16"}, "model.layers.3.mlp.down_proj.scale": {"offset": 5471340, "shape": [1], "dtype": "f32"}, "model.layers.3.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.4.self_attn.q_proj.bits": {"offset": 5471344, "shape": [1296, 24], "dtype": "u32"}, "model.layers.4.self_attn.q_proj.su": {"offset": 5595760, "shape": [576], "dtype": "f16"}, "model.layers.4.self_attn.q_proj.sv": {"offset": 5596912, "shape": [576], "dtype": "f16"}, "model.layers.4.self_attn.q_proj.isc": {"offset": 5598064, "shape": [576], "dtype": "f16"}, "model.layers.4.self_attn.q_proj.osc": {"offset": 5599216, "shape": [576], "dtype": "f16"}, "model.layers.4.self_attn.q_proj.scale": {"offset": 5600368, "shape": [1], "dtype": "f32"}, "model.layers.4.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.4.self_attn.k_proj.bits": {"offset": 5600372, "shape": [432, 24], "dtype": "u32"}, "model.layers.4.self_attn.k_proj.su": {"offset": 5641844, "shape": [576], "dtype": "f16"}, "model.layers.4.self_attn.k_proj.sv": {"offset": 5642996, "shape": [192], "dtype": "f16"}, "model.layers.4.self_attn.k_proj.isc": {"offset": 5643380, "shape": [576], "dtype": "f16"}, "model.layers.4.self_attn.k_proj.osc": {"offset": 5644532, "shape": [192], "dtype": "f16"}, "model.layers.4.self_attn.k_proj.scale": {"offset": 5644916, "shape": [1], "dtype": "f32"}, "model.layers.4.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.4.self_attn.v_proj.bits": {"offset": 5644920, "shape": [432, 24], "dtype": "u32"}, "model.layers.4.self_attn.v_proj.su": {"offset": 5686392, "shape": [576], "dtype": "f16"}, "model.layers.4.self_attn.v_proj.sv": {"offset": 5687544, "shape": [192], "dtype": "f16"}, "model.layers.4.self_attn.v_proj.isc": {"offset": 5687928, "shape": [576], "dtype": "f16"}, "model.layers.4.self_attn.v_proj.osc": {"offset": 5689080, "shape": [192], "dtype": "f16"}, "model.layers.4.self_attn.v_proj.scale": {"offset": 5689464, "shape": [1], "dtype": "f32"}, "model.layers.4.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.4.self_attn.o_proj.bits": {"offset": 5689468, "shape": [1296, 24], "dtype": "u32"}, "model.layers.4.self_attn.o_proj.su": {"offset": 5813884, "shape": [576], "dtype": "f16"}, "model.layers.4.self_attn.o_proj.sv": {"offset": 5815036, "shape": [576], "dtype": "f16"}, "model.layers.4.self_attn.o_proj.isc": {"offset": 5816188, "shape": [576], "dtype": "f16"}, "model.layers.4.self_attn.o_proj.osc": {"offset": 5817340, "shape": [576], "dtype": "f16"}, "model.layers.4.self_attn.o_proj.scale": {"offset": 5818492, "shape": [1], "dtype": "f32"}, "model.layers.4.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.4.mlp.gate_proj.bits": {"offset": 5818496, "shape": [3456, 24], "dtype": "u32"}, "model.layers.4.mlp.gate_proj.su": {"offset": 6150272, "shape": [576], "dtype": "f16"}, "model.layers.4.mlp.gate_proj.sv": {"offset": 6151424, "shape": [1536], "dtype": "f16"}, "model.layers.4.mlp.gate_proj.isc": {"offset": 6154496, "shape": [576], "dtype": "f16"}, "model.layers.4.mlp.gate_proj.osc": {"offset": 6155648, "shape": [1536], "dtype": "f16"}, "model.layers.4.mlp.gate_proj.scale": {"offset": 6158720, "shape": [1], "dtype": "f32"}, "model.layers.4.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.4.mlp.up_proj.bits": {"offset": 6158724, "shape": [3456, 24], "dtype": "u32"}, "model.layers.4.mlp.up_proj.su": {"offset": 6490500, "shape": [576], "dtype": "f16"}, "model.layers.4.mlp.up_proj.sv": {"offset": 6491652, "shape": [1536], "dtype": "f16"}, "model.layers.4.mlp.up_proj.isc": {"offset": 6494724, "shape": [576], "dtype": "f16"}, "model.layers.4.mlp.up_proj.osc": {"offset": 6495876, "shape": [1536], "dtype": "f16"}, "model.layers.4.mlp.up_proj.scale": {"offset": 6498948, "shape": [1], "dtype": "f32"}, "model.layers.4.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.4.mlp.down_proj.bits": {"offset": 6498952, "shape": [3456, 24], "dtype": "u32"}, "model.layers.4.mlp.down_proj.su": {"offset": 6830728, "shape": [1536], "dtype": "f16"}, "model.layers.4.mlp.down_proj.sv": {"offset": 6833800, "shape": [576], "dtype": "f16"}, "model.layers.4.mlp.down_proj.isc": {"offset": 6834952, "shape": [1536], "dtype": "f16"}, "model.layers.4.mlp.down_proj.osc": {"offset": 6838024, "shape": [576], "dtype": "f16"}, "model.layers.4.mlp.down_proj.scale": {"offset": 6839176, "shape": [1], "dtype": "f32"}, "model.layers.4.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.5.self_attn.q_proj.bits": {"offset": 6839180, "shape": [1296, 24], "dtype": "u32"}, "model.layers.5.self_attn.q_proj.su": {"offset": 6963596, "shape": [576], "dtype": "f16"}, "model.layers.5.self_attn.q_proj.sv": {"offset": 6964748, "shape": [576], "dtype": "f16"}, "model.layers.5.self_attn.q_proj.isc": {"offset": 6965900, "shape": [576], "dtype": "f16"}, "model.layers.5.self_attn.q_proj.osc": {"offset": 6967052, "shape": [576], "dtype": "f16"}, "model.layers.5.self_attn.q_proj.scale": {"offset": 6968204, "shape": [1], "dtype": "f32"}, "model.layers.5.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.5.self_attn.k_proj.bits": {"offset": 6968208, "shape": [432, 24], "dtype": "u32"}, "model.layers.5.self_attn.k_proj.su": {"offset": 7009680, "shape": [576], "dtype": "f16"}, "model.layers.5.self_attn.k_proj.sv": {"offset": 7010832, "shape": [192], "dtype": "f16"}, "model.layers.5.self_attn.k_proj.isc": {"offset": 7011216, "shape": [576], "dtype": "f16"}, "model.layers.5.self_attn.k_proj.osc": {"offset": 7012368, "shape": [192], "dtype": "f16"}, "model.layers.5.self_attn.k_proj.scale": {"offset": 7012752, "shape": [1], "dtype": "f32"}, "model.layers.5.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.5.self_attn.v_proj.bits": {"offset": 7012756, "shape": [432, 24], "dtype": "u32"}, "model.layers.5.self_attn.v_proj.su": {"offset": 7054228, "shape": [576], "dtype": "f16"}, "model.layers.5.self_attn.v_proj.sv": {"offset": 7055380, "shape": [192], "dtype": "f16"}, "model.layers.5.self_attn.v_proj.isc": {"offset": 7055764, "shape": [576], "dtype": "f16"}, "model.layers.5.self_attn.v_proj.osc": {"offset": 7056916, "shape": [192], "dtype": "f16"}, "model.layers.5.self_attn.v_proj.scale": {"offset": 7057300, "shape": [1], "dtype": "f32"}, "model.layers.5.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.5.self_attn.o_proj.bits": {"offset": 7057304, "shape": [1296, 24], "dtype": "u32"}, "model.layers.5.self_attn.o_proj.su": {"offset": 7181720, "shape": [576], "dtype": "f16"}, "model.layers.5.self_attn.o_proj.sv": {"offset": 7182872, "shape": [576], "dtype": "f16"}, "model.layers.5.self_attn.o_proj.isc": {"offset": 7184024, "shape": [576], "dtype": "f16"}, "model.layers.5.self_attn.o_proj.osc": {"offset": 7185176, "shape": [576], "dtype": "f16"}, "model.layers.5.self_attn.o_proj.scale": {"offset": 7186328, "shape": [1], "dtype": "f32"}, "model.layers.5.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.5.mlp.gate_proj.bits": {"offset": 7186332, "shape": [3456, 24], "dtype": "u32"}, "model.layers.5.mlp.gate_proj.su": {"offset": 7518108, "shape": [576], "dtype": "f16"}, "model.layers.5.mlp.gate_proj.sv": {"offset": 7519260, "shape": [1536], "dtype": "f16"}, "model.layers.5.mlp.gate_proj.isc": {"offset": 7522332, "shape": [576], "dtype": "f16"}, "model.layers.5.mlp.gate_proj.osc": {"offset": 7523484, "shape": [1536], "dtype": "f16"}, "model.layers.5.mlp.gate_proj.scale": {"offset": 7526556, "shape": [1], "dtype": "f32"}, "model.layers.5.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.5.mlp.up_proj.bits": {"offset": 7526560, "shape": [3456, 24], "dtype": "u32"}, "model.layers.5.mlp.up_proj.su": {"offset": 7858336, "shape": [576], "dtype": "f16"}, "model.layers.5.mlp.up_proj.sv": {"offset": 7859488, "shape": [1536], "dtype": "f16"}, "model.layers.5.mlp.up_proj.isc": {"offset": 7862560, "shape": [576], "dtype": "f16"}, "model.layers.5.mlp.up_proj.osc": {"offset": 7863712, "shape": [1536], "dtype": "f16"}, "model.layers.5.mlp.up_proj.scale": {"offset": 7866784, "shape": [1], "dtype": "f32"}, "model.layers.5.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.5.mlp.down_proj.bits": {"offset": 7866788, "shape": [3456, 24], "dtype": "u32"}, "model.layers.5.mlp.down_proj.su": {"offset": 8198564, "shape": [1536], "dtype": "f16"}, "model.layers.5.mlp.down_proj.sv": {"offset": 8201636, "shape": [576], "dtype": "f16"}, "model.layers.5.mlp.down_proj.isc": {"offset": 8202788, "shape": [1536], "dtype": "f16"}, "model.layers.5.mlp.down_proj.osc": {"offset": 8205860, "shape": [576], "dtype": "f16"}, "model.layers.5.mlp.down_proj.scale": {"offset": 8207012, "shape": [1], "dtype": "f32"}, "model.layers.5.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.6.self_attn.q_proj.bits": {"offset": 8207016, "shape": [1296, 24], "dtype": "u32"}, "model.layers.6.self_attn.q_proj.su": {"offset": 8331432, "shape": [576], "dtype": "f16"}, "model.layers.6.self_attn.q_proj.sv": {"offset": 8332584, "shape": [576], "dtype": "f16"}, "model.layers.6.self_attn.q_proj.isc": {"offset": 8333736, "shape": [576], "dtype": "f16"}, "model.layers.6.self_attn.q_proj.osc": {"offset": 8334888, "shape": [576], "dtype": "f16"}, "model.layers.6.self_attn.q_proj.scale": {"offset": 8336040, "shape": [1], "dtype": "f32"}, "model.layers.6.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.6.self_attn.k_proj.bits": {"offset": 8336044, "shape": [432, 24], "dtype": "u32"}, "model.layers.6.self_attn.k_proj.su": {"offset": 8377516, "shape": [576], "dtype": "f16"}, "model.layers.6.self_attn.k_proj.sv": {"offset": 8378668, "shape": [192], "dtype": "f16"}, "model.layers.6.self_attn.k_proj.isc": {"offset": 8379052, "shape": [576], "dtype": "f16"}, "model.layers.6.self_attn.k_proj.osc": {"offset": 8380204, "shape": [192], "dtype": "f16"}, "model.layers.6.self_attn.k_proj.scale": {"offset": 8380588, "shape": [1], "dtype": "f32"}, "model.layers.6.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.6.self_attn.v_proj.bits": {"offset": 8380592, "shape": [432, 24], "dtype": "u32"}, "model.layers.6.self_attn.v_proj.su": {"offset": 8422064, "shape": [576], "dtype": "f16"}, "model.layers.6.self_attn.v_proj.sv": {"offset": 8423216, "shape": [192], "dtype": "f16"}, "model.layers.6.self_attn.v_proj.isc": {"offset": 8423600, "shape": [576], "dtype": "f16"}, "model.layers.6.self_attn.v_proj.osc": {"offset": 8424752, "shape": [192], "dtype": "f16"}, "model.layers.6.self_attn.v_proj.scale": {"offset": 8425136, "shape": [1], "dtype": "f32"}, "model.layers.6.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.6.self_attn.o_proj.bits": {"offset": 8425140, "shape": [1296, 24], "dtype": "u32"}, "model.layers.6.self_attn.o_proj.su": {"offset": 8549556, "shape": [576], "dtype": "f16"}, "model.layers.6.self_attn.o_proj.sv": {"offset": 8550708, "shape": [576], "dtype": "f16"}, "model.layers.6.self_attn.o_proj.isc": {"offset": 8551860, "shape": [576], "dtype": "f16"}, "model.layers.6.self_attn.o_proj.osc": {"offset": 8553012, "shape": [576], "dtype": "f16"}, "model.layers.6.self_attn.o_proj.scale": {"offset": 8554164, "shape": [1], "dtype": "f32"}, "model.layers.6.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.6.mlp.gate_proj.bits": {"offset": 8554168, "shape": [3456, 24], "dtype": "u32"}, "model.layers.6.mlp.gate_proj.su": {"offset": 8885944, "shape": [576], "dtype": "f16"}, "model.layers.6.mlp.gate_proj.sv": {"offset": 8887096, "shape": [1536], "dtype": "f16"}, "model.layers.6.mlp.gate_proj.isc": {"offset": 8890168, "shape": [576], "dtype": "f16"}, "model.layers.6.mlp.gate_proj.osc": {"offset": 8891320, "shape": [1536], "dtype": "f16"}, "model.layers.6.mlp.gate_proj.scale": {"offset": 8894392, "shape": [1], "dtype": "f32"}, "model.layers.6.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.6.mlp.up_proj.bits": {"offset": 8894396, "shape": [3456, 24], "dtype": "u32"}, "model.layers.6.mlp.up_proj.su": {"offset": 9226172, "shape": [576], "dtype": "f16"}, "model.layers.6.mlp.up_proj.sv": {"offset": 9227324, "shape": [1536], "dtype": "f16"}, "model.layers.6.mlp.up_proj.isc": {"offset": 9230396, "shape": [576], "dtype": "f16"}, "model.layers.6.mlp.up_proj.osc": {"offset": 9231548, "shape": [1536], "dtype": "f16"}, "model.layers.6.mlp.up_proj.scale": {"offset": 9234620, "shape": [1], "dtype": "f32"}, "model.layers.6.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.6.mlp.down_proj.bits": {"offset": 9234624, "shape": [3456, 24], "dtype": "u32"}, "model.layers.6.mlp.down_proj.su": {"offset": 9566400, "shape": [1536], "dtype": "f16"}, "model.layers.6.mlp.down_proj.sv": {"offset": 9569472, "shape": [576], "dtype": "f16"}, "model.layers.6.mlp.down_proj.isc": {"offset": 9570624, "shape": [1536], "dtype": "f16"}, "model.layers.6.mlp.down_proj.osc": {"offset": 9573696, "shape": [576], "dtype": "f16"}, "model.layers.6.mlp.down_proj.scale": {"offset": 9574848, "shape": [1], "dtype": "f32"}, "model.layers.6.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.7.self_attn.q_proj.bits": {"offset": 9574852, "shape": [1296, 24], "dtype": "u32"}, "model.layers.7.self_attn.q_proj.su": {"offset": 9699268, "shape": [576], "dtype": "f16"}, "model.layers.7.self_attn.q_proj.sv": {"offset": 9700420, "shape": [576], "dtype": "f16"}, "model.layers.7.self_attn.q_proj.isc": {"offset": 9701572, "shape": [576], "dtype": "f16"}, "model.layers.7.self_attn.q_proj.osc": {"offset": 9702724, "shape": [576], "dtype": "f16"}, "model.layers.7.self_attn.q_proj.scale": {"offset": 9703876, "shape": [1], "dtype": "f32"}, "model.layers.7.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.7.self_attn.k_proj.bits": {"offset": 9703880, "shape": [432, 24], "dtype": "u32"}, "model.layers.7.self_attn.k_proj.su": {"offset": 9745352, "shape": [576], "dtype": "f16"}, "model.layers.7.self_attn.k_proj.sv": {"offset": 9746504, "shape": [192], "dtype": "f16"}, "model.layers.7.self_attn.k_proj.isc": {"offset": 9746888, "shape": [576], "dtype": "f16"}, "model.layers.7.self_attn.k_proj.osc": {"offset": 9748040, "shape": [192], "dtype": "f16"}, "model.layers.7.self_attn.k_proj.scale": {"offset": 9748424, "shape": [1], "dtype": "f32"}, "model.layers.7.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.7.self_attn.v_proj.bits": {"offset": 9748428, "shape": [432, 24], "dtype": "u32"}, "model.layers.7.self_attn.v_proj.su": {"offset": 9789900, "shape": [576], "dtype": "f16"}, "model.layers.7.self_attn.v_proj.sv": {"offset": 9791052, "shape": [192], "dtype": "f16"}, "model.layers.7.self_attn.v_proj.isc": {"offset": 9791436, "shape": [576], "dtype": "f16"}, "model.layers.7.self_attn.v_proj.osc": {"offset": 9792588, "shape": [192], "dtype": "f16"}, "model.layers.7.self_attn.v_proj.scale": {"offset": 9792972, "shape": [1], "dtype": "f32"}, "model.layers.7.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.7.self_attn.o_proj.bits": {"offset": 9792976, "shape": [1296, 24], "dtype": "u32"}, "model.layers.7.self_attn.o_proj.su": {"offset": 9917392, "shape": [576], "dtype": "f16"}, "model.layers.7.self_attn.o_proj.sv": {"offset": 9918544, "shape": [576], "dtype": "f16"}, "model.layers.7.self_attn.o_proj.isc": {"offset": 9919696, "shape": [576], "dtype": "f16"}, "model.layers.7.self_attn.o_proj.osc": {"offset": 9920848, "shape": [576], "dtype": "f16"}, "model.layers.7.self_attn.o_proj.scale": {"offset": 9922000, "shape": [1], "dtype": "f32"}, "model.layers.7.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.7.mlp.gate_proj.bits": {"offset": 9922004, "shape": [3456, 24], "dtype": "u32"}, "model.layers.7.mlp.gate_proj.su": {"offset": 10253780, "shape": [576], "dtype": "f16"}, "model.layers.7.mlp.gate_proj.sv": {"offset": 10254932, "shape": [1536], "dtype": "f16"}, "model.layers.7.mlp.gate_proj.isc": {"offset": 10258004, "shape": [576], "dtype": "f16"}, "model.layers.7.mlp.gate_proj.osc": {"offset": 10259156, "shape": [1536], "dtype": "f16"}, "model.layers.7.mlp.gate_proj.scale": {"offset": 10262228, "shape": [1], "dtype": "f32"}, "model.layers.7.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.7.mlp.up_proj.bits": {"offset": 10262232, "shape": [3456, 24], "dtype": "u32"}, "model.layers.7.mlp.up_proj.su": {"offset": 10594008, "shape": [576], "dtype": "f16"}, "model.layers.7.mlp.up_proj.sv": {"offset": 10595160, "shape": [1536], "dtype": "f16"}, "model.layers.7.mlp.up_proj.isc": {"offset": 10598232, "shape": [576], "dtype": "f16"}, "model.layers.7.mlp.up_proj.osc": {"offset": 10599384, "shape": [1536], "dtype": "f16"}, "model.layers.7.mlp.up_proj.scale": {"offset": 10602456, "shape": [1], "dtype": "f32"}, "model.layers.7.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.7.mlp.down_proj.bits": {"offset": 10602460, "shape": [3456, 24], "dtype": "u32"}, "model.layers.7.mlp.down_proj.su": {"offset": 10934236, "shape": [1536], "dtype": "f16"}, "model.layers.7.mlp.down_proj.sv": {"offset": 10937308, "shape": [576], "dtype": "f16"}, "model.layers.7.mlp.down_proj.isc": {"offset": 10938460, "shape": [1536], "dtype": "f16"}, "model.layers.7.mlp.down_proj.osc": {"offset": 10941532, "shape": [576], "dtype": "f16"}, "model.layers.7.mlp.down_proj.scale": {"offset": 10942684, "shape": [1], "dtype": "f32"}, "model.layers.7.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.8.self_attn.q_proj.bits": {"offset": 10942688, "shape": [1296, 24], "dtype": "u32"}, "model.layers.8.self_attn.q_proj.su": {"offset": 11067104, "shape": [576], "dtype": "f16"}, "model.layers.8.self_attn.q_proj.sv": {"offset": 11068256, "shape": [576], "dtype": "f16"}, "model.layers.8.self_attn.q_proj.isc": {"offset": 11069408, "shape": [576], "dtype": "f16"}, "model.layers.8.self_attn.q_proj.osc": {"offset": 11070560, "shape": [576], "dtype": "f16"}, "model.layers.8.self_attn.q_proj.scale": {"offset": 11071712, "shape": [1], "dtype": "f32"}, "model.layers.8.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.8.self_attn.k_proj.bits": {"offset": 11071716, "shape": [432, 24], "dtype": "u32"}, "model.layers.8.self_attn.k_proj.su": {"offset": 11113188, "shape": [576], "dtype": "f16"}, "model.layers.8.self_attn.k_proj.sv": {"offset": 11114340, "shape": [192], "dtype": "f16"}, "model.layers.8.self_attn.k_proj.isc": {"offset": 11114724, "shape": [576], "dtype": "f16"}, "model.layers.8.self_attn.k_proj.osc": {"offset": 11115876, "shape": [192], "dtype": "f16"}, "model.layers.8.self_attn.k_proj.scale": {"offset": 11116260, "shape": [1], "dtype": "f32"}, "model.layers.8.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.8.self_attn.v_proj.bits": {"offset": 11116264, "shape": [432, 24], "dtype": "u32"}, "model.layers.8.self_attn.v_proj.su": {"offset": 11157736, "shape": [576], "dtype": "f16"}, "model.layers.8.self_attn.v_proj.sv": {"offset": 11158888, "shape": [192], "dtype": "f16"}, "model.layers.8.self_attn.v_proj.isc": {"offset": 11159272, "shape": [576], "dtype": "f16"}, "model.layers.8.self_attn.v_proj.osc": {"offset": 11160424, "shape": [192], "dtype": "f16"}, "model.layers.8.self_attn.v_proj.scale": {"offset": 11160808, "shape": [1], "dtype": "f32"}, "model.layers.8.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.8.self_attn.o_proj.bits": {"offset": 11160812, "shape": [1296, 24], "dtype": "u32"}, "model.layers.8.self_attn.o_proj.su": {"offset": 11285228, "shape": [576], "dtype": "f16"}, "model.layers.8.self_attn.o_proj.sv": {"offset": 11286380, "shape": [576], "dtype": "f16"}, "model.layers.8.self_attn.o_proj.isc": {"offset": 11287532, "shape": [576], "dtype": "f16"}, "model.layers.8.self_attn.o_proj.osc": {"offset": 11288684, "shape": [576], "dtype": "f16"}, "model.layers.8.self_attn.o_proj.scale": {"offset": 11289836, "shape": [1], "dtype": "f32"}, "model.layers.8.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.8.mlp.gate_proj.bits": {"offset": 11289840, "shape": [3456, 24], "dtype": "u32"}, "model.layers.8.mlp.gate_proj.su": {"offset": 11621616, "shape": [576], "dtype": "f16"}, "model.layers.8.mlp.gate_proj.sv": {"offset": 11622768, "shape": [1536], "dtype": "f16"}, "model.layers.8.mlp.gate_proj.isc": {"offset": 11625840, "shape": [576], "dtype": "f16"}, "model.layers.8.mlp.gate_proj.osc": {"offset": 11626992, "shape": [1536], "dtype": "f16"}, "model.layers.8.mlp.gate_proj.scale": {"offset": 11630064, "shape": [1], "dtype": "f32"}, "model.layers.8.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.8.mlp.up_proj.bits": {"offset": 11630068, "shape": [3456, 24], "dtype": "u32"}, "model.layers.8.mlp.up_proj.su": {"offset": 11961844, "shape": [576], "dtype": "f16"}, "model.layers.8.mlp.up_proj.sv": {"offset": 11962996, "shape": [1536], "dtype": "f16"}, "model.layers.8.mlp.up_proj.isc": {"offset": 11966068, "shape": [576], "dtype": "f16"}, "model.layers.8.mlp.up_proj.osc": {"offset": 11967220, "shape": [1536], "dtype": "f16"}, "model.layers.8.mlp.up_proj.scale": {"offset": 11970292, "shape": [1], "dtype": "f32"}, "model.layers.8.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.8.mlp.down_proj.bits": {"offset": 11970296, "shape": [3456, 24], "dtype": "u32"}, "model.layers.8.mlp.down_proj.su": {"offset": 12302072, "shape": [1536], "dtype": "f16"}, "model.layers.8.mlp.down_proj.sv": {"offset": 12305144, "shape": [576], "dtype": "f16"}, "model.layers.8.mlp.down_proj.isc": {"offset": 12306296, "shape": [1536], "dtype": "f16"}, "model.layers.8.mlp.down_proj.osc": {"offset": 12309368, "shape": [576], "dtype": "f16"}, "model.layers.8.mlp.down_proj.scale": {"offset": 12310520, "shape": [1], "dtype": "f32"}, "model.layers.8.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.9.self_attn.q_proj.bits": {"offset": 12310524, "shape": [1296, 24], "dtype": "u32"}, "model.layers.9.self_attn.q_proj.su": {"offset": 12434940, "shape": [576], "dtype": "f16"}, "model.layers.9.self_attn.q_proj.sv": {"offset": 12436092, "shape": [576], "dtype": "f16"}, "model.layers.9.self_attn.q_proj.isc": {"offset": 12437244, "shape": [576], "dtype": "f16"}, "model.layers.9.self_attn.q_proj.osc": {"offset": 12438396, "shape": [576], "dtype": "f16"}, "model.layers.9.self_attn.q_proj.scale": {"offset": 12439548, "shape": [1], "dtype": "f32"}, "model.layers.9.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.9.self_attn.k_proj.bits": {"offset": 12439552, "shape": [432, 24], "dtype": "u32"}, "model.layers.9.self_attn.k_proj.su": {"offset": 12481024, "shape": [576], "dtype": "f16"}, "model.layers.9.self_attn.k_proj.sv": {"offset": 12482176, "shape": [192], "dtype": "f16"}, "model.layers.9.self_attn.k_proj.isc": {"offset": 12482560, "shape": [576], "dtype": "f16"}, "model.layers.9.self_attn.k_proj.osc": {"offset": 12483712, "shape": [192], "dtype": "f16"}, "model.layers.9.self_attn.k_proj.scale": {"offset": 12484096, "shape": [1], "dtype": "f32"}, "model.layers.9.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.9.self_attn.v_proj.bits": {"offset": 12484100, "shape": [432, 24], "dtype": "u32"}, "model.layers.9.self_attn.v_proj.su": {"offset": 12525572, "shape": [576], "dtype": "f16"}, "model.layers.9.self_attn.v_proj.sv": {"offset": 12526724, "shape": [192], "dtype": "f16"}, "model.layers.9.self_attn.v_proj.isc": {"offset": 12527108, "shape": [576], "dtype": "f16"}, "model.layers.9.self_attn.v_proj.osc": {"offset": 12528260, "shape": [192], "dtype": "f16"}, "model.layers.9.self_attn.v_proj.scale": {"offset": 12528644, "shape": [1], "dtype": "f32"}, "model.layers.9.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.9.self_attn.o_proj.bits": {"offset": 12528648, "shape": [1296, 24], "dtype": "u32"}, "model.layers.9.self_attn.o_proj.su": {"offset": 12653064, "shape": [576], "dtype": "f16"}, "model.layers.9.self_attn.o_proj.sv": {"offset": 12654216, "shape": [576], "dtype": "f16"}, "model.layers.9.self_attn.o_proj.isc": {"offset": 12655368, "shape": [576], "dtype": "f16"}, "model.layers.9.self_attn.o_proj.osc": {"offset": 12656520, "shape": [576], "dtype": "f16"}, "model.layers.9.self_attn.o_proj.scale": {"offset": 12657672, "shape": [1], "dtype": "f32"}, "model.layers.9.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.9.mlp.gate_proj.bits": {"offset": 12657676, "shape": [3456, 24], "dtype": "u32"}, "model.layers.9.mlp.gate_proj.su": {"offset": 12989452, "shape": [576], "dtype": "f16"}, "model.layers.9.mlp.gate_proj.sv": {"offset": 12990604, "shape": [1536], "dtype": "f16"}, "model.layers.9.mlp.gate_proj.isc": {"offset": 12993676, "shape": [576], "dtype": "f16"}, "model.layers.9.mlp.gate_proj.osc": {"offset": 12994828, "shape": [1536], "dtype": "f16"}, "model.layers.9.mlp.gate_proj.scale": {"offset": 12997900, "shape": [1], "dtype": "f32"}, "model.layers.9.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.9.mlp.up_proj.bits": {"offset": 12997904, "shape": [3456, 24], "dtype": "u32"}, "model.layers.9.mlp.up_proj.su": {"offset": 13329680, "shape": [576], "dtype": "f16"}, "model.layers.9.mlp.up_proj.sv": {"offset": 13330832, "shape": [1536], "dtype": "f16"}, "model.layers.9.mlp.up_proj.isc": {"offset": 13333904, "shape": [576], "dtype": "f16"}, "model.layers.9.mlp.up_proj.osc": {"offset": 13335056, "shape": [1536], "dtype": "f16"}, "model.layers.9.mlp.up_proj.scale": {"offset": 13338128, "shape": [1], "dtype": "f32"}, "model.layers.9.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.9.mlp.down_proj.bits": {"offset": 13338132, "shape": [3456, 24], "dtype": "u32"}, "model.layers.9.mlp.down_proj.su": {"offset": 13669908, "shape": [1536], "dtype": "f16"}, "model.layers.9.mlp.down_proj.sv": {"offset": 13672980, "shape": [576], "dtype": "f16"}, "model.layers.9.mlp.down_proj.isc": {"offset": 13674132, "shape": [1536], "dtype": "f16"}, "model.layers.9.mlp.down_proj.osc": {"offset": 13677204, "shape": [576], "dtype": "f16"}, "model.layers.9.mlp.down_proj.scale": {"offset": 13678356, "shape": [1], "dtype": "f32"}, "model.layers.9.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.10.self_attn.q_proj.bits": {"offset": 13678360, "shape": [1296, 24], "dtype": "u32"}, "model.layers.10.self_attn.q_proj.su": {"offset": 13802776, "shape": [576], "dtype": "f16"}, "model.layers.10.self_attn.q_proj.sv": {"offset": 13803928, "shape": [576], "dtype": "f16"}, "model.layers.10.self_attn.q_proj.isc": {"offset": 13805080, "shape": [576], "dtype": "f16"}, "model.layers.10.self_attn.q_proj.osc": {"offset": 13806232, "shape": [576], "dtype": "f16"}, "model.layers.10.self_attn.q_proj.scale": {"offset": 13807384, "shape": [1], "dtype": "f32"}, "model.layers.10.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.10.self_attn.k_proj.bits": {"offset": 13807388, "shape": [432, 24], "dtype": "u32"}, "model.layers.10.self_attn.k_proj.su": {"offset": 13848860, "shape": [576], "dtype": "f16"}, "model.layers.10.self_attn.k_proj.sv": {"offset": 13850012, "shape": [192], "dtype": "f16"}, "model.layers.10.self_attn.k_proj.isc": {"offset": 13850396, "shape": [576], "dtype": "f16"}, "model.layers.10.self_attn.k_proj.osc": {"offset": 13851548, "shape": [192], "dtype": "f16"}, "model.layers.10.self_attn.k_proj.scale": {"offset": 13851932, "shape": [1], "dtype": "f32"}, "model.layers.10.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.10.self_attn.v_proj.bits": {"offset": 13851936, "shape": [432, 24], "dtype": "u32"}, "model.layers.10.self_attn.v_proj.su": {"offset": 13893408, "shape": [576], "dtype": "f16"}, "model.layers.10.self_attn.v_proj.sv": {"offset": 13894560, "shape": [192], "dtype": "f16"}, "model.layers.10.self_attn.v_proj.isc": {"offset": 13894944, "shape": [576], "dtype": "f16"}, "model.layers.10.self_attn.v_proj.osc": {"offset": 13896096, "shape": [192], "dtype": "f16"}, "model.layers.10.self_attn.v_proj.scale": {"offset": 13896480, "shape": [1], "dtype": "f32"}, "model.layers.10.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.10.self_attn.o_proj.bits": {"offset": 13896484, "shape": [1296, 24], "dtype": "u32"}, "model.layers.10.self_attn.o_proj.su": {"offset": 14020900, "shape": [576], "dtype": "f16"}, "model.layers.10.self_attn.o_proj.sv": {"offset": 14022052, "shape": [576], "dtype": "f16"}, "model.layers.10.self_attn.o_proj.isc": {"offset": 14023204, "shape": [576], "dtype": "f16"}, "model.layers.10.self_attn.o_proj.osc": {"offset": 14024356, "shape": [576], "dtype": "f16"}, "model.layers.10.self_attn.o_proj.scale": {"offset": 14025508, "shape": [1], "dtype": "f32"}, "model.layers.10.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.10.mlp.gate_proj.bits": {"offset": 14025512, "shape": [3456, 24], "dtype": "u32"}, "model.layers.10.mlp.gate_proj.su": {"offset": 14357288, "shape": [576], "dtype": "f16"}, "model.layers.10.mlp.gate_proj.sv": {"offset": 14358440, "shape": [1536], "dtype": "f16"}, "model.layers.10.mlp.gate_proj.isc": {"offset": 14361512, "shape": [576], "dtype": "f16"}, "model.layers.10.mlp.gate_proj.osc": {"offset": 14362664, "shape": [1536], "dtype": "f16"}, "model.layers.10.mlp.gate_proj.scale": {"offset": 14365736, "shape": [1], "dtype": "f32"}, "model.layers.10.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.10.mlp.up_proj.bits": {"offset": 14365740, "shape": [3456, 24], "dtype": "u32"}, "model.layers.10.mlp.up_proj.su": {"offset": 14697516, "shape": [576], "dtype": "f16"}, "model.layers.10.mlp.up_proj.sv": {"offset": 14698668, "shape": [1536], "dtype": "f16"}, "model.layers.10.mlp.up_proj.isc": {"offset": 14701740, "shape": [576], "dtype": "f16"}, "model.layers.10.mlp.up_proj.osc": {"offset": 14702892, "shape": [1536], "dtype": "f16"}, "model.layers.10.mlp.up_proj.scale": {"offset": 14705964, "shape": [1], "dtype": "f32"}, "model.layers.10.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.10.mlp.down_proj.bits": {"offset": 14705968, "shape": [3456, 24], "dtype": "u32"}, "model.layers.10.mlp.down_proj.su": {"offset": 15037744, "shape": [1536], "dtype": "f16"}, "model.layers.10.mlp.down_proj.sv": {"offset": 15040816, "shape": [576], "dtype": "f16"}, "model.layers.10.mlp.down_proj.isc": {"offset": 15041968, "shape": [1536], "dtype": "f16"}, "model.layers.10.mlp.down_proj.osc": {"offset": 15045040, "shape": [576], "dtype": "f16"}, "model.layers.10.mlp.down_proj.scale": {"offset": 15046192, "shape": [1], "dtype": "f32"}, "model.layers.10.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.11.self_attn.q_proj.bits": {"offset": 15046196, "shape": [1296, 24], "dtype": "u32"}, "model.layers.11.self_attn.q_proj.su": {"offset": 15170612, "shape": [576], "dtype": "f16"}, "model.layers.11.self_attn.q_proj.sv": {"offset": 15171764, "shape": [576], "dtype": "f16"}, "model.layers.11.self_attn.q_proj.isc": {"offset": 15172916, "shape": [576], "dtype": "f16"}, "model.layers.11.self_attn.q_proj.osc": {"offset": 15174068, "shape": [576], "dtype": "f16"}, "model.layers.11.self_attn.q_proj.scale": {"offset": 15175220, "shape": [1], "dtype": "f32"}, "model.layers.11.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.11.self_attn.k_proj.bits": {"offset": 15175224, "shape": [432, 24], "dtype": "u32"}, "model.layers.11.self_attn.k_proj.su": {"offset": 15216696, "shape": [576], "dtype": "f16"}, "model.layers.11.self_attn.k_proj.sv": {"offset": 15217848, "shape": [192], "dtype": "f16"}, "model.layers.11.self_attn.k_proj.isc": {"offset": 15218232, "shape": [576], "dtype": "f16"}, "model.layers.11.self_attn.k_proj.osc": {"offset": 15219384, "shape": [192], "dtype": "f16"}, "model.layers.11.self_attn.k_proj.scale": {"offset": 15219768, "shape": [1], "dtype": "f32"}, "model.layers.11.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.11.self_attn.v_proj.bits": {"offset": 15219772, "shape": [432, 24], "dtype": "u32"}, "model.layers.11.self_attn.v_proj.su": {"offset": 15261244, "shape": [576], "dtype": "f16"}, "model.layers.11.self_attn.v_proj.sv": {"offset": 15262396, "shape": [192], "dtype": "f16"}, "model.layers.11.self_attn.v_proj.isc": {"offset": 15262780, "shape": [576], "dtype": "f16"}, "model.layers.11.self_attn.v_proj.osc": {"offset": 15263932, "shape": [192], "dtype": "f16"}, "model.layers.11.self_attn.v_proj.scale": {"offset": 15264316, "shape": [1], "dtype": "f32"}, "model.layers.11.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.11.self_attn.o_proj.bits": {"offset": 15264320, "shape": [1296, 24], "dtype": "u32"}, "model.layers.11.self_attn.o_proj.su": {"offset": 15388736, "shape": [576], "dtype": "f16"}, "model.layers.11.self_attn.o_proj.sv": {"offset": 15389888, "shape": [576], "dtype": "f16"}, "model.layers.11.self_attn.o_proj.isc": {"offset": 15391040, "shape": [576], "dtype": "f16"}, "model.layers.11.self_attn.o_proj.osc": {"offset": 15392192, "shape": [576], "dtype": "f16"}, "model.layers.11.self_attn.o_proj.scale": {"offset": 15393344, "shape": [1], "dtype": "f32"}, "model.layers.11.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.11.mlp.gate_proj.bits": {"offset": 15393348, "shape": [3456, 24], "dtype": "u32"}, "model.layers.11.mlp.gate_proj.su": {"offset": 15725124, "shape": [576], "dtype": "f16"}, "model.layers.11.mlp.gate_proj.sv": {"offset": 15726276, "shape": [1536], "dtype": "f16"}, "model.layers.11.mlp.gate_proj.isc": {"offset": 15729348, "shape": [576], "dtype": "f16"}, "model.layers.11.mlp.gate_proj.osc": {"offset": 15730500, "shape": [1536], "dtype": "f16"}, "model.layers.11.mlp.gate_proj.scale": {"offset": 15733572, "shape": [1], "dtype": "f32"}, "model.layers.11.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.11.mlp.up_proj.bits": {"offset": 15733576, "shape": [3456, 24], "dtype": "u32"}, "model.layers.11.mlp.up_proj.su": {"offset": 16065352, "shape": [576], "dtype": "f16"}, "model.layers.11.mlp.up_proj.sv": {"offset": 16066504, "shape": [1536], "dtype": "f16"}, "model.layers.11.mlp.up_proj.isc": {"offset": 16069576, "shape": [576], "dtype": "f16"}, "model.layers.11.mlp.up_proj.osc": {"offset": 16070728, "shape": [1536], "dtype": "f16"}, "model.layers.11.mlp.up_proj.scale": {"offset": 16073800, "shape": [1], "dtype": "f32"}, "model.layers.11.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.11.mlp.down_proj.bits": {"offset": 16073804, "shape": [3456, 24], "dtype": "u32"}, "model.layers.11.mlp.down_proj.su": {"offset": 16405580, "shape": [1536], "dtype": "f16"}, "model.layers.11.mlp.down_proj.sv": {"offset": 16408652, "shape": [576], "dtype": "f16"}, "model.layers.11.mlp.down_proj.isc": {"offset": 16409804, "shape": [1536], "dtype": "f16"}, "model.layers.11.mlp.down_proj.osc": {"offset": 16412876, "shape": [576], "dtype": "f16"}, "model.layers.11.mlp.down_proj.scale": {"offset": 16414028, "shape": [1], "dtype": "f32"}, "model.layers.11.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.12.self_attn.q_proj.bits": {"offset": 16414032, "shape": [1296, 24], "dtype": "u32"}, "model.layers.12.self_attn.q_proj.su": {"offset": 16538448, "shape": [576], "dtype": "f16"}, "model.layers.12.self_attn.q_proj.sv": {"offset": 16539600, "shape": [576], "dtype": "f16"}, "model.layers.12.self_attn.q_proj.isc": {"offset": 16540752, "shape": [576], "dtype": "f16"}, "model.layers.12.self_attn.q_proj.osc": {"offset": 16541904, "shape": [576], "dtype": "f16"}, "model.layers.12.self_attn.q_proj.scale": {"offset": 16543056, "shape": [1], "dtype": "f32"}, "model.layers.12.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.12.self_attn.k_proj.bits": {"offset": 16543060, "shape": [432, 24], "dtype": "u32"}, "model.layers.12.self_attn.k_proj.su": {"offset": 16584532, "shape": [576], "dtype": "f16"}, "model.layers.12.self_attn.k_proj.sv": {"offset": 16585684, "shape": [192], "dtype": "f16"}, "model.layers.12.self_attn.k_proj.isc": {"offset": 16586068, "shape": [576], "dtype": "f16"}, "model.layers.12.self_attn.k_proj.osc": {"offset": 16587220, "shape": [192], "dtype": "f16"}, "model.layers.12.self_attn.k_proj.scale": {"offset": 16587604, "shape": [1], "dtype": "f32"}, "model.layers.12.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.12.self_attn.v_proj.bits": {"offset": 16587608, "shape": [432, 24], "dtype": "u32"}, "model.layers.12.self_attn.v_proj.su": {"offset": 16629080, "shape": [576], "dtype": "f16"}, "model.layers.12.self_attn.v_proj.sv": {"offset": 16630232, "shape": [192], "dtype": "f16"}, "model.layers.12.self_attn.v_proj.isc": {"offset": 16630616, "shape": [576], "dtype": "f16"}, "model.layers.12.self_attn.v_proj.osc": {"offset": 16631768, "shape": [192], "dtype": "f16"}, "model.layers.12.self_attn.v_proj.scale": {"offset": 16632152, "shape": [1], "dtype": "f32"}, "model.layers.12.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.12.self_attn.o_proj.bits": {"offset": 16632156, "shape": [1296, 24], "dtype": "u32"}, "model.layers.12.self_attn.o_proj.su": {"offset": 16756572, "shape": [576], "dtype": "f16"}, "model.layers.12.self_attn.o_proj.sv": {"offset": 16757724, "shape": [576], "dtype": "f16"}, "model.layers.12.self_attn.o_proj.isc": {"offset": 16758876, "shape": [576], "dtype": "f16"}, "model.layers.12.self_attn.o_proj.osc": {"offset": 16760028, "shape": [576], "dtype": "f16"}, "model.layers.12.self_attn.o_proj.scale": {"offset": 16761180, "shape": [1], "dtype": "f32"}, "model.layers.12.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.12.mlp.gate_proj.bits": {"offset": 16761184, "shape": [3456, 24], "dtype": "u32"}, "model.layers.12.mlp.gate_proj.su": {"offset": 17092960, "shape": [576], "dtype": "f16"}, "model.layers.12.mlp.gate_proj.sv": {"offset": 17094112, "shape": [1536], "dtype": "f16"}, "model.layers.12.mlp.gate_proj.isc": {"offset": 17097184, "shape": [576], "dtype": "f16"}, "model.layers.12.mlp.gate_proj.osc": {"offset": 17098336, "shape": [1536], "dtype": "f16"}, "model.layers.12.mlp.gate_proj.scale": {"offset": 17101408, "shape": [1], "dtype": "f32"}, "model.layers.12.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.12.mlp.up_proj.bits": {"offset": 17101412, "shape": [3456, 24], "dtype": "u32"}, "model.layers.12.mlp.up_proj.su": {"offset": 17433188, "shape": [576], "dtype": "f16"}, "model.layers.12.mlp.up_proj.sv": {"offset": 17434340, "shape": [1536], "dtype": "f16"}, "model.layers.12.mlp.up_proj.isc": {"offset": 17437412, "shape": [576], "dtype": "f16"}, "model.layers.12.mlp.up_proj.osc": {"offset": 17438564, "shape": [1536], "dtype": "f16"}, "model.layers.12.mlp.up_proj.scale": {"offset": 17441636, "shape": [1], "dtype": "f32"}, "model.layers.12.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.12.mlp.down_proj.bits": {"offset": 17441640, "shape": [3456, 24], "dtype": "u32"}, "model.layers.12.mlp.down_proj.su": {"offset": 17773416, "shape": [1536], "dtype": "f16"}, "model.layers.12.mlp.down_proj.sv": {"offset": 17776488, "shape": [576], "dtype": "f16"}, "model.layers.12.mlp.down_proj.isc": {"offset": 17777640, "shape": [1536], "dtype": "f16"}, "model.layers.12.mlp.down_proj.osc": {"offset": 17780712, "shape": [576], "dtype": "f16"}, "model.layers.12.mlp.down_proj.scale": {"offset": 17781864, "shape": [1], "dtype": "f32"}, "model.layers.12.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.13.self_attn.q_proj.bits": {"offset": 17781868, "shape": [1296, 24], "dtype": "u32"}, "model.layers.13.self_attn.q_proj.su": {"offset": 17906284, "shape": [576], "dtype": "f16"}, "model.layers.13.self_attn.q_proj.sv": {"offset": 17907436, "shape": [576], "dtype": "f16"}, "model.layers.13.self_attn.q_proj.isc": {"offset": 17908588, "shape": [576], "dtype": "f16"}, "model.layers.13.self_attn.q_proj.osc": {"offset": 17909740, "shape": [576], "dtype": "f16"}, "model.layers.13.self_attn.q_proj.scale": {"offset": 17910892, "shape": [1], "dtype": "f32"}, "model.layers.13.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.13.self_attn.k_proj.bits": {"offset": 17910896, "shape": [432, 24], "dtype": "u32"}, "model.layers.13.self_attn.k_proj.su": {"offset": 17952368, "shape": [576], "dtype": "f16"}, "model.layers.13.self_attn.k_proj.sv": {"offset": 17953520, "shape": [192], "dtype": "f16"}, "model.layers.13.self_attn.k_proj.isc": {"offset": 17953904, "shape": [576], "dtype": "f16"}, "model.layers.13.self_attn.k_proj.osc": {"offset": 17955056, "shape": [192], "dtype": "f16"}, "model.layers.13.self_attn.k_proj.scale": {"offset": 17955440, "shape": [1], "dtype": "f32"}, "model.layers.13.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.13.self_attn.v_proj.bits": {"offset": 17955444, "shape": [432, 24], "dtype": "u32"}, "model.layers.13.self_attn.v_proj.su": {"offset": 17996916, "shape": [576], "dtype": "f16"}, "model.layers.13.self_attn.v_proj.sv": {"offset": 17998068, "shape": [192], "dtype": "f16"}, "model.layers.13.self_attn.v_proj.isc": {"offset": 17998452, "shape": [576], "dtype": "f16"}, "model.layers.13.self_attn.v_proj.osc": {"offset": 17999604, "shape": [192], "dtype": "f16"}, "model.layers.13.self_attn.v_proj.scale": {"offset": 17999988, "shape": [1], "dtype": "f32"}, "model.layers.13.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.13.self_attn.o_proj.bits": {"offset": 17999992, "shape": [1296, 24], "dtype": "u32"}, "model.layers.13.self_attn.o_proj.su": {"offset": 18124408, "shape": [576], "dtype": "f16"}, "model.layers.13.self_attn.o_proj.sv": {"offset": 18125560, "shape": [576], "dtype": "f16"}, "model.layers.13.self_attn.o_proj.isc": {"offset": 18126712, "shape": [576], "dtype": "f16"}, "model.layers.13.self_attn.o_proj.osc": {"offset": 18127864, "shape": [576], "dtype": "f16"}, "model.layers.13.self_attn.o_proj.scale": {"offset": 18129016, "shape": [1], "dtype": "f32"}, "model.layers.13.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.13.mlp.gate_proj.bits": {"offset": 18129020, "shape": [3456, 24], "dtype": "u32"}, "model.layers.13.mlp.gate_proj.su": {"offset": 18460796, "shape": [576], "dtype": "f16"}, "model.layers.13.mlp.gate_proj.sv": {"offset": 18461948, "shape": [1536], "dtype": "f16"}, "model.layers.13.mlp.gate_proj.isc": {"offset": 18465020, "shape": [576], "dtype": "f16"}, "model.layers.13.mlp.gate_proj.osc": {"offset": 18466172, "shape": [1536], "dtype": "f16"}, "model.layers.13.mlp.gate_proj.scale": {"offset": 18469244, "shape": [1], "dtype": "f32"}, "model.layers.13.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.13.mlp.up_proj.bits": {"offset": 18469248, "shape": [3456, 24], "dtype": "u32"}, "model.layers.13.mlp.up_proj.su": {"offset": 18801024, "shape": [576], "dtype": "f16"}, "model.layers.13.mlp.up_proj.sv": {"offset": 18802176, "shape": [1536], "dtype": "f16"}, "model.layers.13.mlp.up_proj.isc": {"offset": 18805248, "shape": [576], "dtype": "f16"}, "model.layers.13.mlp.up_proj.osc": {"offset": 18806400, "shape": [1536], "dtype": "f16"}, "model.layers.13.mlp.up_proj.scale": {"offset": 18809472, "shape": [1], "dtype": "f32"}, "model.layers.13.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.13.mlp.down_proj.bits": {"offset": 18809476, "shape": [3456, 24], "dtype": "u32"}, "model.layers.13.mlp.down_proj.su": {"offset": 19141252, "shape": [1536], "dtype": "f16"}, "model.layers.13.mlp.down_proj.sv": {"offset": 19144324, "shape": [576], "dtype": "f16"}, "model.layers.13.mlp.down_proj.isc": {"offset": 19145476, "shape": [1536], "dtype": "f16"}, "model.layers.13.mlp.down_proj.osc": {"offset": 19148548, "shape": [576], "dtype": "f16"}, "model.layers.13.mlp.down_proj.scale": {"offset": 19149700, "shape": [1], "dtype": "f32"}, "model.layers.13.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.14.self_attn.q_proj.bits": {"offset": 19149704, "shape": [1296, 24], "dtype": "u32"}, "model.layers.14.self_attn.q_proj.su": {"offset": 19274120, "shape": [576], "dtype": "f16"}, "model.layers.14.self_attn.q_proj.sv": {"offset": 19275272, "shape": [576], "dtype": "f16"}, "model.layers.14.self_attn.q_proj.isc": {"offset": 19276424, "shape": [576], "dtype": "f16"}, "model.layers.14.self_attn.q_proj.osc": {"offset": 19277576, "shape": [576], "dtype": "f16"}, "model.layers.14.self_attn.q_proj.scale": {"offset": 19278728, "shape": [1], "dtype": "f32"}, "model.layers.14.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.14.self_attn.k_proj.bits": {"offset": 19278732, "shape": [432, 24], "dtype": "u32"}, "model.layers.14.self_attn.k_proj.su": {"offset": 19320204, "shape": [576], "dtype": "f16"}, "model.layers.14.self_attn.k_proj.sv": {"offset": 19321356, "shape": [192], "dtype": "f16"}, "model.layers.14.self_attn.k_proj.isc": {"offset": 19321740, "shape": [576], "dtype": "f16"}, "model.layers.14.self_attn.k_proj.osc": {"offset": 19322892, "shape": [192], "dtype": "f16"}, "model.layers.14.self_attn.k_proj.scale": {"offset": 19323276, "shape": [1], "dtype": "f32"}, "model.layers.14.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.14.self_attn.v_proj.bits": {"offset": 19323280, "shape": [432, 24], "dtype": "u32"}, "model.layers.14.self_attn.v_proj.su": {"offset": 19364752, "shape": [576], "dtype": "f16"}, "model.layers.14.self_attn.v_proj.sv": {"offset": 19365904, "shape": [192], "dtype": "f16"}, "model.layers.14.self_attn.v_proj.isc": {"offset": 19366288, "shape": [576], "dtype": "f16"}, "model.layers.14.self_attn.v_proj.osc": {"offset": 19367440, "shape": [192], "dtype": "f16"}, "model.layers.14.self_attn.v_proj.scale": {"offset": 19367824, "shape": [1], "dtype": "f32"}, "model.layers.14.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.14.self_attn.o_proj.bits": {"offset": 19367828, "shape": [1296, 24], "dtype": "u32"}, "model.layers.14.self_attn.o_proj.su": {"offset": 19492244, "shape": [576], "dtype": "f16"}, "model.layers.14.self_attn.o_proj.sv": {"offset": 19493396, "shape": [576], "dtype": "f16"}, "model.layers.14.self_attn.o_proj.isc": {"offset": 19494548, "shape": [576], "dtype": "f16"}, "model.layers.14.self_attn.o_proj.osc": {"offset": 19495700, "shape": [576], "dtype": "f16"}, "model.layers.14.self_attn.o_proj.scale": {"offset": 19496852, "shape": [1], "dtype": "f32"}, "model.layers.14.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.14.mlp.gate_proj.bits": {"offset": 19496856, "shape": [3456, 24], "dtype": "u32"}, "model.layers.14.mlp.gate_proj.su": {"offset": 19828632, "shape": [576], "dtype": "f16"}, "model.layers.14.mlp.gate_proj.sv": {"offset": 19829784, "shape": [1536], "dtype": "f16"}, "model.layers.14.mlp.gate_proj.isc": {"offset": 19832856, "shape": [576], "dtype": "f16"}, "model.layers.14.mlp.gate_proj.osc": {"offset": 19834008, "shape": [1536], "dtype": "f16"}, "model.layers.14.mlp.gate_proj.scale": {"offset": 19837080, "shape": [1], "dtype": "f32"}, "model.layers.14.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.14.mlp.up_proj.bits": {"offset": 19837084, "shape": [3456, 24], "dtype": "u32"}, "model.layers.14.mlp.up_proj.su": {"offset": 20168860, "shape": [576], "dtype": "f16"}, "model.layers.14.mlp.up_proj.sv": {"offset": 20170012, "shape": [1536], "dtype": "f16"}, "model.layers.14.mlp.up_proj.isc": {"offset": 20173084, "shape": [576], "dtype": "f16"}, "model.layers.14.mlp.up_proj.osc": {"offset": 20174236, "shape": [1536], "dtype": "f16"}, "model.layers.14.mlp.up_proj.scale": {"offset": 20177308, "shape": [1], "dtype": "f32"}, "model.layers.14.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.14.mlp.down_proj.bits": {"offset": 20177312, "shape": [3456, 24], "dtype": "u32"}, "model.layers.14.mlp.down_proj.su": {"offset": 20509088, "shape": [1536], "dtype": "f16"}, "model.layers.14.mlp.down_proj.sv": {"offset": 20512160, "shape": [576], "dtype": "f16"}, "model.layers.14.mlp.down_proj.isc": {"offset": 20513312, "shape": [1536], "dtype": "f16"}, "model.layers.14.mlp.down_proj.osc": {"offset": 20516384, "shape": [576], "dtype": "f16"}, "model.layers.14.mlp.down_proj.scale": {"offset": 20517536, "shape": [1], "dtype": "f32"}, "model.layers.14.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.15.self_attn.q_proj.bits": {"offset": 20517540, "shape": [1296, 24], "dtype": "u32"}, "model.layers.15.self_attn.q_proj.su": {"offset": 20641956, "shape": [576], "dtype": "f16"}, "model.layers.15.self_attn.q_proj.sv": {"offset": 20643108, "shape": [576], "dtype": "f16"}, "model.layers.15.self_attn.q_proj.isc": {"offset": 20644260, "shape": [576], "dtype": "f16"}, "model.layers.15.self_attn.q_proj.osc": {"offset": 20645412, "shape": [576], "dtype": "f16"}, "model.layers.15.self_attn.q_proj.scale": {"offset": 20646564, "shape": [1], "dtype": "f32"}, "model.layers.15.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.15.self_attn.k_proj.bits": {"offset": 20646568, "shape": [432, 24], "dtype": "u32"}, "model.layers.15.self_attn.k_proj.su": {"offset": 20688040, "shape": [576], "dtype": "f16"}, "model.layers.15.self_attn.k_proj.sv": {"offset": 20689192, "shape": [192], "dtype": "f16"}, "model.layers.15.self_attn.k_proj.isc": {"offset": 20689576, "shape": [576], "dtype": "f16"}, "model.layers.15.self_attn.k_proj.osc": {"offset": 20690728, "shape": [192], "dtype": "f16"}, "model.layers.15.self_attn.k_proj.scale": {"offset": 20691112, "shape": [1], "dtype": "f32"}, "model.layers.15.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.15.self_attn.v_proj.bits": {"offset": 20691116, "shape": [432, 24], "dtype": "u32"}, "model.layers.15.self_attn.v_proj.su": {"offset": 20732588, "shape": [576], "dtype": "f16"}, "model.layers.15.self_attn.v_proj.sv": {"offset": 20733740, "shape": [192], "dtype": "f16"}, "model.layers.15.self_attn.v_proj.isc": {"offset": 20734124, "shape": [576], "dtype": "f16"}, "model.layers.15.self_attn.v_proj.osc": {"offset": 20735276, "shape": [192], "dtype": "f16"}, "model.layers.15.self_attn.v_proj.scale": {"offset": 20735660, "shape": [1], "dtype": "f32"}, "model.layers.15.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.15.self_attn.o_proj.bits": {"offset": 20735664, "shape": [1296, 24], "dtype": "u32"}, "model.layers.15.self_attn.o_proj.su": {"offset": 20860080, "shape": [576], "dtype": "f16"}, "model.layers.15.self_attn.o_proj.sv": {"offset": 20861232, "shape": [576], "dtype": "f16"}, "model.layers.15.self_attn.o_proj.isc": {"offset": 20862384, "shape": [576], "dtype": "f16"}, "model.layers.15.self_attn.o_proj.osc": {"offset": 20863536, "shape": [576], "dtype": "f16"}, "model.layers.15.self_attn.o_proj.scale": {"offset": 20864688, "shape": [1], "dtype": "f32"}, "model.layers.15.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.15.mlp.gate_proj.bits": {"offset": 20864692, "shape": [3456, 24], "dtype": "u32"}, "model.layers.15.mlp.gate_proj.su": {"offset": 21196468, "shape": [576], "dtype": "f16"}, "model.layers.15.mlp.gate_proj.sv": {"offset": 21197620, "shape": [1536], "dtype": "f16"}, "model.layers.15.mlp.gate_proj.isc": {"offset": 21200692, "shape": [576], "dtype": "f16"}, "model.layers.15.mlp.gate_proj.osc": {"offset": 21201844, "shape": [1536], "dtype": "f16"}, "model.layers.15.mlp.gate_proj.scale": {"offset": 21204916, "shape": [1], "dtype": "f32"}, "model.layers.15.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.15.mlp.up_proj.bits": {"offset": 21204920, "shape": [3456, 24], "dtype": "u32"}, "model.layers.15.mlp.up_proj.su": {"offset": 21536696, "shape": [576], "dtype": "f16"}, "model.layers.15.mlp.up_proj.sv": {"offset": 21537848, "shape": [1536], "dtype": "f16"}, "model.layers.15.mlp.up_proj.isc": {"offset": 21540920, "shape": [576], "dtype": "f16"}, "model.layers.15.mlp.up_proj.osc": {"offset": 21542072, "shape": [1536], "dtype": "f16"}, "model.layers.15.mlp.up_proj.scale": {"offset": 21545144, "shape": [1], "dtype": "f32"}, "model.layers.15.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.15.mlp.down_proj.bits": {"offset": 21545148, "shape": [3456, 24], "dtype": "u32"}, "model.layers.15.mlp.down_proj.su": {"offset": 21876924, "shape": [1536], "dtype": "f16"}, "model.layers.15.mlp.down_proj.sv": {"offset": 21879996, "shape": [576], "dtype": "f16"}, "model.layers.15.mlp.down_proj.isc": {"offset": 21881148, "shape": [1536], "dtype": "f16"}, "model.layers.15.mlp.down_proj.osc": {"offset": 21884220, "shape": [576], "dtype": "f16"}, "model.layers.15.mlp.down_proj.scale": {"offset": 21885372, "shape": [1], "dtype": "f32"}, "model.layers.15.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.16.self_attn.q_proj.bits": {"offset": 21885376, "shape": [1296, 24], "dtype": "u32"}, "model.layers.16.self_attn.q_proj.su": {"offset": 22009792, "shape": [576], "dtype": "f16"}, "model.layers.16.self_attn.q_proj.sv": {"offset": 22010944, "shape": [576], "dtype": "f16"}, "model.layers.16.self_attn.q_proj.isc": {"offset": 22012096, "shape": [576], "dtype": "f16"}, "model.layers.16.self_attn.q_proj.osc": {"offset": 22013248, "shape": [576], "dtype": "f16"}, "model.layers.16.self_attn.q_proj.scale": {"offset": 22014400, "shape": [1], "dtype": "f32"}, "model.layers.16.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.16.self_attn.k_proj.bits": {"offset": 22014404, "shape": [432, 24], "dtype": "u32"}, "model.layers.16.self_attn.k_proj.su": {"offset": 22055876, "shape": [576], "dtype": "f16"}, "model.layers.16.self_attn.k_proj.sv": {"offset": 22057028, "shape": [192], "dtype": "f16"}, "model.layers.16.self_attn.k_proj.isc": {"offset": 22057412, "shape": [576], "dtype": "f16"}, "model.layers.16.self_attn.k_proj.osc": {"offset": 22058564, "shape": [192], "dtype": "f16"}, "model.layers.16.self_attn.k_proj.scale": {"offset": 22058948, "shape": [1], "dtype": "f32"}, "model.layers.16.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.16.self_attn.v_proj.bits": {"offset": 22058952, "shape": [432, 24], "dtype": "u32"}, "model.layers.16.self_attn.v_proj.su": {"offset": 22100424, "shape": [576], "dtype": "f16"}, "model.layers.16.self_attn.v_proj.sv": {"offset": 22101576, "shape": [192], "dtype": "f16"}, "model.layers.16.self_attn.v_proj.isc": {"offset": 22101960, "shape": [576], "dtype": "f16"}, "model.layers.16.self_attn.v_proj.osc": {"offset": 22103112, "shape": [192], "dtype": "f16"}, "model.layers.16.self_attn.v_proj.scale": {"offset": 22103496, "shape": [1], "dtype": "f32"}, "model.layers.16.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.16.self_attn.o_proj.bits": {"offset": 22103500, "shape": [1296, 24], "dtype": "u32"}, "model.layers.16.self_attn.o_proj.su": {"offset": 22227916, "shape": [576], "dtype": "f16"}, "model.layers.16.self_attn.o_proj.sv": {"offset": 22229068, "shape": [576], "dtype": "f16"}, "model.layers.16.self_attn.o_proj.isc": {"offset": 22230220, "shape": [576], "dtype": "f16"}, "model.layers.16.self_attn.o_proj.osc": {"offset": 22231372, "shape": [576], "dtype": "f16"}, "model.layers.16.self_attn.o_proj.scale": {"offset": 22232524, "shape": [1], "dtype": "f32"}, "model.layers.16.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.16.mlp.gate_proj.bits": {"offset": 22232528, "shape": [3456, 24], "dtype": "u32"}, "model.layers.16.mlp.gate_proj.su": {"offset": 22564304, "shape": [576], "dtype": "f16"}, "model.layers.16.mlp.gate_proj.sv": {"offset": 22565456, "shape": [1536], "dtype": "f16"}, "model.layers.16.mlp.gate_proj.isc": {"offset": 22568528, "shape": [576], "dtype": "f16"}, "model.layers.16.mlp.gate_proj.osc": {"offset": 22569680, "shape": [1536], "dtype": "f16"}, "model.layers.16.mlp.gate_proj.scale": {"offset": 22572752, "shape": [1], "dtype": "f32"}, "model.layers.16.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.16.mlp.up_proj.bits": {"offset": 22572756, "shape": [3456, 24], "dtype": "u32"}, "model.layers.16.mlp.up_proj.su": {"offset": 22904532, "shape": [576], "dtype": "f16"}, "model.layers.16.mlp.up_proj.sv": {"offset": 22905684, "shape": [1536], "dtype": "f16"}, "model.layers.16.mlp.up_proj.isc": {"offset": 22908756, "shape": [576], "dtype": "f16"}, "model.layers.16.mlp.up_proj.osc": {"offset": 22909908, "shape": [1536], "dtype": "f16"}, "model.layers.16.mlp.up_proj.scale": {"offset": 22912980, "shape": [1], "dtype": "f32"}, "model.layers.16.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.16.mlp.down_proj.bits": {"offset": 22912984, "shape": [3456, 24], "dtype": "u32"}, "model.layers.16.mlp.down_proj.su": {"offset": 23244760, "shape": [1536], "dtype": "f16"}, "model.layers.16.mlp.down_proj.sv": {"offset": 23247832, "shape": [576], "dtype": "f16"}, "model.layers.16.mlp.down_proj.isc": {"offset": 23248984, "shape": [1536], "dtype": "f16"}, "model.layers.16.mlp.down_proj.osc": {"offset": 23252056, "shape": [576], "dtype": "f16"}, "model.layers.16.mlp.down_proj.scale": {"offset": 23253208, "shape": [1], "dtype": "f32"}, "model.layers.16.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.17.self_attn.q_proj.bits": {"offset": 23253212, "shape": [1296, 24], "dtype": "u32"}, "model.layers.17.self_attn.q_proj.su": {"offset": 23377628, "shape": [576], "dtype": "f16"}, "model.layers.17.self_attn.q_proj.sv": {"offset": 23378780, "shape": [576], "dtype": "f16"}, "model.layers.17.self_attn.q_proj.isc": {"offset": 23379932, "shape": [576], "dtype": "f16"}, "model.layers.17.self_attn.q_proj.osc": {"offset": 23381084, "shape": [576], "dtype": "f16"}, "model.layers.17.self_attn.q_proj.scale": {"offset": 23382236, "shape": [1], "dtype": "f32"}, "model.layers.17.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.17.self_attn.k_proj.bits": {"offset": 23382240, "shape": [432, 24], "dtype": "u32"}, "model.layers.17.self_attn.k_proj.su": {"offset": 23423712, "shape": [576], "dtype": "f16"}, "model.layers.17.self_attn.k_proj.sv": {"offset": 23424864, "shape": [192], "dtype": "f16"}, "model.layers.17.self_attn.k_proj.isc": {"offset": 23425248, "shape": [576], "dtype": "f16"}, "model.layers.17.self_attn.k_proj.osc": {"offset": 23426400, "shape": [192], "dtype": "f16"}, "model.layers.17.self_attn.k_proj.scale": {"offset": 23426784, "shape": [1], "dtype": "f32"}, "model.layers.17.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.17.self_attn.v_proj.bits": {"offset": 23426788, "shape": [432, 24], "dtype": "u32"}, "model.layers.17.self_attn.v_proj.su": {"offset": 23468260, "shape": [576], "dtype": "f16"}, "model.layers.17.self_attn.v_proj.sv": {"offset": 23469412, "shape": [192], "dtype": "f16"}, "model.layers.17.self_attn.v_proj.isc": {"offset": 23469796, "shape": [576], "dtype": "f16"}, "model.layers.17.self_attn.v_proj.osc": {"offset": 23470948, "shape": [192], "dtype": "f16"}, "model.layers.17.self_attn.v_proj.scale": {"offset": 23471332, "shape": [1], "dtype": "f32"}, "model.layers.17.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.17.self_attn.o_proj.bits": {"offset": 23471336, "shape": [1296, 24], "dtype": "u32"}, "model.layers.17.self_attn.o_proj.su": {"offset": 23595752, "shape": [576], "dtype": "f16"}, "model.layers.17.self_attn.o_proj.sv": {"offset": 23596904, "shape": [576], "dtype": "f16"}, "model.layers.17.self_attn.o_proj.isc": {"offset": 23598056, "shape": [576], "dtype": "f16"}, "model.layers.17.self_attn.o_proj.osc": {"offset": 23599208, "shape": [576], "dtype": "f16"}, "model.layers.17.self_attn.o_proj.scale": {"offset": 23600360, "shape": [1], "dtype": "f32"}, "model.layers.17.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.17.mlp.gate_proj.bits": {"offset": 23600364, "shape": [3456, 24], "dtype": "u32"}, "model.layers.17.mlp.gate_proj.su": {"offset": 23932140, "shape": [576], "dtype": "f16"}, "model.layers.17.mlp.gate_proj.sv": {"offset": 23933292, "shape": [1536], "dtype": "f16"}, "model.layers.17.mlp.gate_proj.isc": {"offset": 23936364, "shape": [576], "dtype": "f16"}, "model.layers.17.mlp.gate_proj.osc": {"offset": 23937516, "shape": [1536], "dtype": "f16"}, "model.layers.17.mlp.gate_proj.scale": {"offset": 23940588, "shape": [1], "dtype": "f32"}, "model.layers.17.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.17.mlp.up_proj.bits": {"offset": 23940592, "shape": [3456, 24], "dtype": "u32"}, "model.layers.17.mlp.up_proj.su": {"offset": 24272368, "shape": [576], "dtype": "f16"}, "model.layers.17.mlp.up_proj.sv": {"offset": 24273520, "shape": [1536], "dtype": "f16"}, "model.layers.17.mlp.up_proj.isc": {"offset": 24276592, "shape": [576], "dtype": "f16"}, "model.layers.17.mlp.up_proj.osc": {"offset": 24277744, "shape": [1536], "dtype": "f16"}, "model.layers.17.mlp.up_proj.scale": {"offset": 24280816, "shape": [1], "dtype": "f32"}, "model.layers.17.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.17.mlp.down_proj.bits": {"offset": 24280820, "shape": [3456, 24], "dtype": "u32"}, "model.layers.17.mlp.down_proj.su": {"offset": 24612596, "shape": [1536], "dtype": "f16"}, "model.layers.17.mlp.down_proj.sv": {"offset": 24615668, "shape": [576], "dtype": "f16"}, "model.layers.17.mlp.down_proj.isc": {"offset": 24616820, "shape": [1536], "dtype": "f16"}, "model.layers.17.mlp.down_proj.osc": {"offset": 24619892, "shape": [576], "dtype": "f16"}, "model.layers.17.mlp.down_proj.scale": {"offset": 24621044, "shape": [1], "dtype": "f32"}, "model.layers.17.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.18.self_attn.q_proj.bits": {"offset": 24621048, "shape": [1296, 24], "dtype": "u32"}, "model.layers.18.self_attn.q_proj.su": {"offset": 24745464, "shape": [576], "dtype": "f16"}, "model.layers.18.self_attn.q_proj.sv": {"offset": 24746616, "shape": [576], "dtype": "f16"}, "model.layers.18.self_attn.q_proj.isc": {"offset": 24747768, "shape": [576], "dtype": "f16"}, "model.layers.18.self_attn.q_proj.osc": {"offset": 24748920, "shape": [576], "dtype": "f16"}, "model.layers.18.self_attn.q_proj.scale": {"offset": 24750072, "shape": [1], "dtype": "f32"}, "model.layers.18.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.18.self_attn.k_proj.bits": {"offset": 24750076, "shape": [432, 24], "dtype": "u32"}, "model.layers.18.self_attn.k_proj.su": {"offset": 24791548, "shape": [576], "dtype": "f16"}, "model.layers.18.self_attn.k_proj.sv": {"offset": 24792700, "shape": [192], "dtype": "f16"}, "model.layers.18.self_attn.k_proj.isc": {"offset": 24793084, "shape": [576], "dtype": "f16"}, "model.layers.18.self_attn.k_proj.osc": {"offset": 24794236, "shape": [192], "dtype": "f16"}, "model.layers.18.self_attn.k_proj.scale": {"offset": 24794620, "shape": [1], "dtype": "f32"}, "model.layers.18.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.18.self_attn.v_proj.bits": {"offset": 24794624, "shape": [432, 24], "dtype": "u32"}, "model.layers.18.self_attn.v_proj.su": {"offset": 24836096, "shape": [576], "dtype": "f16"}, "model.layers.18.self_attn.v_proj.sv": {"offset": 24837248, "shape": [192], "dtype": "f16"}, "model.layers.18.self_attn.v_proj.isc": {"offset": 24837632, "shape": [576], "dtype": "f16"}, "model.layers.18.self_attn.v_proj.osc": {"offset": 24838784, "shape": [192], "dtype": "f16"}, "model.layers.18.self_attn.v_proj.scale": {"offset": 24839168, "shape": [1], "dtype": "f32"}, "model.layers.18.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.18.self_attn.o_proj.bits": {"offset": 24839172, "shape": [1296, 24], "dtype": "u32"}, "model.layers.18.self_attn.o_proj.su": {"offset": 24963588, "shape": [576], "dtype": "f16"}, "model.layers.18.self_attn.o_proj.sv": {"offset": 24964740, "shape": [576], "dtype": "f16"}, "model.layers.18.self_attn.o_proj.isc": {"offset": 24965892, "shape": [576], "dtype": "f16"}, "model.layers.18.self_attn.o_proj.osc": {"offset": 24967044, "shape": [576], "dtype": "f16"}, "model.layers.18.self_attn.o_proj.scale": {"offset": 24968196, "shape": [1], "dtype": "f32"}, "model.layers.18.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.18.mlp.gate_proj.bits": {"offset": 24968200, "shape": [3456, 24], "dtype": "u32"}, "model.layers.18.mlp.gate_proj.su": {"offset": 25299976, "shape": [576], "dtype": "f16"}, "model.layers.18.mlp.gate_proj.sv": {"offset": 25301128, "shape": [1536], "dtype": "f16"}, "model.layers.18.mlp.gate_proj.isc": {"offset": 25304200, "shape": [576], "dtype": "f16"}, "model.layers.18.mlp.gate_proj.osc": {"offset": 25305352, "shape": [1536], "dtype": "f16"}, "model.layers.18.mlp.gate_proj.scale": {"offset": 25308424, "shape": [1], "dtype": "f32"}, "model.layers.18.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.18.mlp.up_proj.bits": {"offset": 25308428, "shape": [3456, 24], "dtype": "u32"}, "model.layers.18.mlp.up_proj.su": {"offset": 25640204, "shape": [576], "dtype": "f16"}, "model.layers.18.mlp.up_proj.sv": {"offset": 25641356, "shape": [1536], "dtype": "f16"}, "model.layers.18.mlp.up_proj.isc": {"offset": 25644428, "shape": [576], "dtype": "f16"}, "model.layers.18.mlp.up_proj.osc": {"offset": 25645580, "shape": [1536], "dtype": "f16"}, "model.layers.18.mlp.up_proj.scale": {"offset": 25648652, "shape": [1], "dtype": "f32"}, "model.layers.18.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.18.mlp.down_proj.bits": {"offset": 25648656, "shape": [3456, 24], "dtype": "u32"}, "model.layers.18.mlp.down_proj.su": {"offset": 25980432, "shape": [1536], "dtype": "f16"}, "model.layers.18.mlp.down_proj.sv": {"offset": 25983504, "shape": [576], "dtype": "f16"}, "model.layers.18.mlp.down_proj.isc": {"offset": 25984656, "shape": [1536], "dtype": "f16"}, "model.layers.18.mlp.down_proj.osc": {"offset": 25987728, "shape": [576], "dtype": "f16"}, "model.layers.18.mlp.down_proj.scale": {"offset": 25988880, "shape": [1], "dtype": "f32"}, "model.layers.18.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.19.self_attn.q_proj.bits": {"offset": 25988884, "shape": [1296, 24], "dtype": "u32"}, "model.layers.19.self_attn.q_proj.su": {"offset": 26113300, "shape": [576], "dtype": "f16"}, "model.layers.19.self_attn.q_proj.sv": {"offset": 26114452, "shape": [576], "dtype": "f16"}, "model.layers.19.self_attn.q_proj.isc": {"offset": 26115604, "shape": [576], "dtype": "f16"}, "model.layers.19.self_attn.q_proj.osc": {"offset": 26116756, "shape": [576], "dtype": "f16"}, "model.layers.19.self_attn.q_proj.scale": {"offset": 26117908, "shape": [1], "dtype": "f32"}, "model.layers.19.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.19.self_attn.k_proj.bits": {"offset": 26117912, "shape": [432, 24], "dtype": "u32"}, "model.layers.19.self_attn.k_proj.su": {"offset": 26159384, "shape": [576], "dtype": "f16"}, "model.layers.19.self_attn.k_proj.sv": {"offset": 26160536, "shape": [192], "dtype": "f16"}, "model.layers.19.self_attn.k_proj.isc": {"offset": 26160920, "shape": [576], "dtype": "f16"}, "model.layers.19.self_attn.k_proj.osc": {"offset": 26162072, "shape": [192], "dtype": "f16"}, "model.layers.19.self_attn.k_proj.scale": {"offset": 26162456, "shape": [1], "dtype": "f32"}, "model.layers.19.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.19.self_attn.v_proj.bits": {"offset": 26162460, "shape": [432, 24], "dtype": "u32"}, "model.layers.19.self_attn.v_proj.su": {"offset": 26203932, "shape": [576], "dtype": "f16"}, "model.layers.19.self_attn.v_proj.sv": {"offset": 26205084, "shape": [192], "dtype": "f16"}, "model.layers.19.self_attn.v_proj.isc": {"offset": 26205468, "shape": [576], "dtype": "f16"}, "model.layers.19.self_attn.v_proj.osc": {"offset": 26206620, "shape": [192], "dtype": "f16"}, "model.layers.19.self_attn.v_proj.scale": {"offset": 26207004, "shape": [1], "dtype": "f32"}, "model.layers.19.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.19.self_attn.o_proj.bits": {"offset": 26207008, "shape": [1296, 24], "dtype": "u32"}, "model.layers.19.self_attn.o_proj.su": {"offset": 26331424, "shape": [576], "dtype": "f16"}, "model.layers.19.self_attn.o_proj.sv": {"offset": 26332576, "shape": [576], "dtype": "f16"}, "model.layers.19.self_attn.o_proj.isc": {"offset": 26333728, "shape": [576], "dtype": "f16"}, "model.layers.19.self_attn.o_proj.osc": {"offset": 26334880, "shape": [576], "dtype": "f16"}, "model.layers.19.self_attn.o_proj.scale": {"offset": 26336032, "shape": [1], "dtype": "f32"}, "model.layers.19.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.19.mlp.gate_proj.bits": {"offset": 26336036, "shape": [3456, 24], "dtype": "u32"}, "model.layers.19.mlp.gate_proj.su": {"offset": 26667812, "shape": [576], "dtype": "f16"}, "model.layers.19.mlp.gate_proj.sv": {"offset": 26668964, "shape": [1536], "dtype": "f16"}, "model.layers.19.mlp.gate_proj.isc": {"offset": 26672036, "shape": [576], "dtype": "f16"}, "model.layers.19.mlp.gate_proj.osc": {"offset": 26673188, "shape": [1536], "dtype": "f16"}, "model.layers.19.mlp.gate_proj.scale": {"offset": 26676260, "shape": [1], "dtype": "f32"}, "model.layers.19.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.19.mlp.up_proj.bits": {"offset": 26676264, "shape": [3456, 24], "dtype": "u32"}, "model.layers.19.mlp.up_proj.su": {"offset": 27008040, "shape": [576], "dtype": "f16"}, "model.layers.19.mlp.up_proj.sv": {"offset": 27009192, "shape": [1536], "dtype": "f16"}, "model.layers.19.mlp.up_proj.isc": {"offset": 27012264, "shape": [576], "dtype": "f16"}, "model.layers.19.mlp.up_proj.osc": {"offset": 27013416, "shape": [1536], "dtype": "f16"}, "model.layers.19.mlp.up_proj.scale": {"offset": 27016488, "shape": [1], "dtype": "f32"}, "model.layers.19.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.19.mlp.down_proj.bits": {"offset": 27016492, "shape": [3456, 24], "dtype": "u32"}, "model.layers.19.mlp.down_proj.su": {"offset": 27348268, "shape": [1536], "dtype": "f16"}, "model.layers.19.mlp.down_proj.sv": {"offset": 27351340, "shape": [576], "dtype": "f16"}, "model.layers.19.mlp.down_proj.isc": {"offset": 27352492, "shape": [1536], "dtype": "f16"}, "model.layers.19.mlp.down_proj.osc": {"offset": 27355564, "shape": [576], "dtype": "f16"}, "model.layers.19.mlp.down_proj.scale": {"offset": 27356716, "shape": [1], "dtype": "f32"}, "model.layers.19.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.20.self_attn.q_proj.bits": {"offset": 27356720, "shape": [1296, 24], "dtype": "u32"}, "model.layers.20.self_attn.q_proj.su": {"offset": 27481136, "shape": [576], "dtype": "f16"}, "model.layers.20.self_attn.q_proj.sv": {"offset": 27482288, "shape": [576], "dtype": "f16"}, "model.layers.20.self_attn.q_proj.isc": {"offset": 27483440, "shape": [576], "dtype": "f16"}, "model.layers.20.self_attn.q_proj.osc": {"offset": 27484592, "shape": [576], "dtype": "f16"}, "model.layers.20.self_attn.q_proj.scale": {"offset": 27485744, "shape": [1], "dtype": "f32"}, "model.layers.20.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.20.self_attn.k_proj.bits": {"offset": 27485748, "shape": [432, 24], "dtype": "u32"}, "model.layers.20.self_attn.k_proj.su": {"offset": 27527220, "shape": [576], "dtype": "f16"}, "model.layers.20.self_attn.k_proj.sv": {"offset": 27528372, "shape": [192], "dtype": "f16"}, "model.layers.20.self_attn.k_proj.isc": {"offset": 27528756, "shape": [576], "dtype": "f16"}, "model.layers.20.self_attn.k_proj.osc": {"offset": 27529908, "shape": [192], "dtype": "f16"}, "model.layers.20.self_attn.k_proj.scale": {"offset": 27530292, "shape": [1], "dtype": "f32"}, "model.layers.20.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.20.self_attn.v_proj.bits": {"offset": 27530296, "shape": [432, 24], "dtype": "u32"}, "model.layers.20.self_attn.v_proj.su": {"offset": 27571768, "shape": [576], "dtype": "f16"}, "model.layers.20.self_attn.v_proj.sv": {"offset": 27572920, "shape": [192], "dtype": "f16"}, "model.layers.20.self_attn.v_proj.isc": {"offset": 27573304, "shape": [576], "dtype": "f16"}, "model.layers.20.self_attn.v_proj.osc": {"offset": 27574456, "shape": [192], "dtype": "f16"}, "model.layers.20.self_attn.v_proj.scale": {"offset": 27574840, "shape": [1], "dtype": "f32"}, "model.layers.20.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.20.self_attn.o_proj.bits": {"offset": 27574844, "shape": [1296, 24], "dtype": "u32"}, "model.layers.20.self_attn.o_proj.su": {"offset": 27699260, "shape": [576], "dtype": "f16"}, "model.layers.20.self_attn.o_proj.sv": {"offset": 27700412, "shape": [576], "dtype": "f16"}, "model.layers.20.self_attn.o_proj.isc": {"offset": 27701564, "shape": [576], "dtype": "f16"}, "model.layers.20.self_attn.o_proj.osc": {"offset": 27702716, "shape": [576], "dtype": "f16"}, "model.layers.20.self_attn.o_proj.scale": {"offset": 27703868, "shape": [1], "dtype": "f32"}, "model.layers.20.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.20.mlp.gate_proj.bits": {"offset": 27703872, "shape": [3456, 24], "dtype": "u32"}, "model.layers.20.mlp.gate_proj.su": {"offset": 28035648, "shape": [576], "dtype": "f16"}, "model.layers.20.mlp.gate_proj.sv": {"offset": 28036800, "shape": [1536], "dtype": "f16"}, "model.layers.20.mlp.gate_proj.isc": {"offset": 28039872, "shape": [576], "dtype": "f16"}, "model.layers.20.mlp.gate_proj.osc": {"offset": 28041024, "shape": [1536], "dtype": "f16"}, "model.layers.20.mlp.gate_proj.scale": {"offset": 28044096, "shape": [1], "dtype": "f32"}, "model.layers.20.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.20.mlp.up_proj.bits": {"offset": 28044100, "shape": [3456, 24], "dtype": "u32"}, "model.layers.20.mlp.up_proj.su": {"offset": 28375876, "shape": [576], "dtype": "f16"}, "model.layers.20.mlp.up_proj.sv": {"offset": 28377028, "shape": [1536], "dtype": "f16"}, "model.layers.20.mlp.up_proj.isc": {"offset": 28380100, "shape": [576], "dtype": "f16"}, "model.layers.20.mlp.up_proj.osc": {"offset": 28381252, "shape": [1536], "dtype": "f16"}, "model.layers.20.mlp.up_proj.scale": {"offset": 28384324, "shape": [1], "dtype": "f32"}, "model.layers.20.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.20.mlp.down_proj.bits": {"offset": 28384328, "shape": [3456, 24], "dtype": "u32"}, "model.layers.20.mlp.down_proj.su": {"offset": 28716104, "shape": [1536], "dtype": "f16"}, "model.layers.20.mlp.down_proj.sv": {"offset": 28719176, "shape": [576], "dtype": "f16"}, "model.layers.20.mlp.down_proj.isc": {"offset": 28720328, "shape": [1536], "dtype": "f16"}, "model.layers.20.mlp.down_proj.osc": {"offset": 28723400, "shape": [576], "dtype": "f16"}, "model.layers.20.mlp.down_proj.scale": {"offset": 28724552, "shape": [1], "dtype": "f32"}, "model.layers.20.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.21.self_attn.q_proj.bits": {"offset": 28724556, "shape": [1296, 24], "dtype": "u32"}, "model.layers.21.self_attn.q_proj.su": {"offset": 28848972, "shape": [576], "dtype": "f16"}, "model.layers.21.self_attn.q_proj.sv": {"offset": 28850124, "shape": [576], "dtype": "f16"}, "model.layers.21.self_attn.q_proj.isc": {"offset": 28851276, "shape": [576], "dtype": "f16"}, "model.layers.21.self_attn.q_proj.osc": {"offset": 28852428, "shape": [576], "dtype": "f16"}, "model.layers.21.self_attn.q_proj.scale": {"offset": 28853580, "shape": [1], "dtype": "f32"}, "model.layers.21.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.21.self_attn.k_proj.bits": {"offset": 28853584, "shape": [432, 24], "dtype": "u32"}, "model.layers.21.self_attn.k_proj.su": {"offset": 28895056, "shape": [576], "dtype": "f16"}, "model.layers.21.self_attn.k_proj.sv": {"offset": 28896208, "shape": [192], "dtype": "f16"}, "model.layers.21.self_attn.k_proj.isc": {"offset": 28896592, "shape": [576], "dtype": "f16"}, "model.layers.21.self_attn.k_proj.osc": {"offset": 28897744, "shape": [192], "dtype": "f16"}, "model.layers.21.self_attn.k_proj.scale": {"offset": 28898128, "shape": [1], "dtype": "f32"}, "model.layers.21.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.21.self_attn.v_proj.bits": {"offset": 28898132, "shape": [432, 24], "dtype": "u32"}, "model.layers.21.self_attn.v_proj.su": {"offset": 28939604, "shape": [576], "dtype": "f16"}, "model.layers.21.self_attn.v_proj.sv": {"offset": 28940756, "shape": [192], "dtype": "f16"}, "model.layers.21.self_attn.v_proj.isc": {"offset": 28941140, "shape": [576], "dtype": "f16"}, "model.layers.21.self_attn.v_proj.osc": {"offset": 28942292, "shape": [192], "dtype": "f16"}, "model.layers.21.self_attn.v_proj.scale": {"offset": 28942676, "shape": [1], "dtype": "f32"}, "model.layers.21.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.21.self_attn.o_proj.bits": {"offset": 28942680, "shape": [1296, 24], "dtype": "u32"}, "model.layers.21.self_attn.o_proj.su": {"offset": 29067096, "shape": [576], "dtype": "f16"}, "model.layers.21.self_attn.o_proj.sv": {"offset": 29068248, "shape": [576], "dtype": "f16"}, "model.layers.21.self_attn.o_proj.isc": {"offset": 29069400, "shape": [576], "dtype": "f16"}, "model.layers.21.self_attn.o_proj.osc": {"offset": 29070552, "shape": [576], "dtype": "f16"}, "model.layers.21.self_attn.o_proj.scale": {"offset": 29071704, "shape": [1], "dtype": "f32"}, "model.layers.21.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.21.mlp.gate_proj.bits": {"offset": 29071708, "shape": [3456, 24], "dtype": "u32"}, "model.layers.21.mlp.gate_proj.su": {"offset": 29403484, "shape": [576], "dtype": "f16"}, "model.layers.21.mlp.gate_proj.sv": {"offset": 29404636, "shape": [1536], "dtype": "f16"}, "model.layers.21.mlp.gate_proj.isc": {"offset": 29407708, "shape": [576], "dtype": "f16"}, "model.layers.21.mlp.gate_proj.osc": {"offset": 29408860, "shape": [1536], "dtype": "f16"}, "model.layers.21.mlp.gate_proj.scale": {"offset": 29411932, "shape": [1], "dtype": "f32"}, "model.layers.21.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.21.mlp.up_proj.bits": {"offset": 29411936, "shape": [3456, 24], "dtype": "u32"}, "model.layers.21.mlp.up_proj.su": {"offset": 29743712, "shape": [576], "dtype": "f16"}, "model.layers.21.mlp.up_proj.sv": {"offset": 29744864, "shape": [1536], "dtype": "f16"}, "model.layers.21.mlp.up_proj.isc": {"offset": 29747936, "shape": [576], "dtype": "f16"}, "model.layers.21.mlp.up_proj.osc": {"offset": 29749088, "shape": [1536], "dtype": "f16"}, "model.layers.21.mlp.up_proj.scale": {"offset": 29752160, "shape": [1], "dtype": "f32"}, "model.layers.21.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.21.mlp.down_proj.bits": {"offset": 29752164, "shape": [3456, 24], "dtype": "u32"}, "model.layers.21.mlp.down_proj.su": {"offset": 30083940, "shape": [1536], "dtype": "f16"}, "model.layers.21.mlp.down_proj.sv": {"offset": 30087012, "shape": [576], "dtype": "f16"}, "model.layers.21.mlp.down_proj.isc": {"offset": 30088164, "shape": [1536], "dtype": "f16"}, "model.layers.21.mlp.down_proj.osc": {"offset": 30091236, "shape": [576], "dtype": "f16"}, "model.layers.21.mlp.down_proj.scale": {"offset": 30092388, "shape": [1], "dtype": "f32"}, "model.layers.21.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.22.self_attn.q_proj.bits": {"offset": 30092392, "shape": [1296, 24], "dtype": "u32"}, "model.layers.22.self_attn.q_proj.su": {"offset": 30216808, "shape": [576], "dtype": "f16"}, "model.layers.22.self_attn.q_proj.sv": {"offset": 30217960, "shape": [576], "dtype": "f16"}, "model.layers.22.self_attn.q_proj.isc": {"offset": 30219112, "shape": [576], "dtype": "f16"}, "model.layers.22.self_attn.q_proj.osc": {"offset": 30220264, "shape": [576], "dtype": "f16"}, "model.layers.22.self_attn.q_proj.scale": {"offset": 30221416, "shape": [1], "dtype": "f32"}, "model.layers.22.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.22.self_attn.k_proj.bits": {"offset": 30221420, "shape": [432, 24], "dtype": "u32"}, "model.layers.22.self_attn.k_proj.su": {"offset": 30262892, "shape": [576], "dtype": "f16"}, "model.layers.22.self_attn.k_proj.sv": {"offset": 30264044, "shape": [192], "dtype": "f16"}, "model.layers.22.self_attn.k_proj.isc": {"offset": 30264428, "shape": [576], "dtype": "f16"}, "model.layers.22.self_attn.k_proj.osc": {"offset": 30265580, "shape": [192], "dtype": "f16"}, "model.layers.22.self_attn.k_proj.scale": {"offset": 30265964, "shape": [1], "dtype": "f32"}, "model.layers.22.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.22.self_attn.v_proj.bits": {"offset": 30265968, "shape": [432, 24], "dtype": "u32"}, "model.layers.22.self_attn.v_proj.su": {"offset": 30307440, "shape": [576], "dtype": "f16"}, "model.layers.22.self_attn.v_proj.sv": {"offset": 30308592, "shape": [192], "dtype": "f16"}, "model.layers.22.self_attn.v_proj.isc": {"offset": 30308976, "shape": [576], "dtype": "f16"}, "model.layers.22.self_attn.v_proj.osc": {"offset": 30310128, "shape": [192], "dtype": "f16"}, "model.layers.22.self_attn.v_proj.scale": {"offset": 30310512, "shape": [1], "dtype": "f32"}, "model.layers.22.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.22.self_attn.o_proj.bits": {"offset": 30310516, "shape": [1296, 24], "dtype": "u32"}, "model.layers.22.self_attn.o_proj.su": {"offset": 30434932, "shape": [576], "dtype": "f16"}, "model.layers.22.self_attn.o_proj.sv": {"offset": 30436084, "shape": [576], "dtype": "f16"}, "model.layers.22.self_attn.o_proj.isc": {"offset": 30437236, "shape": [576], "dtype": "f16"}, "model.layers.22.self_attn.o_proj.osc": {"offset": 30438388, "shape": [576], "dtype": "f16"}, "model.layers.22.self_attn.o_proj.scale": {"offset": 30439540, "shape": [1], "dtype": "f32"}, "model.layers.22.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.22.mlp.gate_proj.bits": {"offset": 30439544, "shape": [3456, 24], "dtype": "u32"}, "model.layers.22.mlp.gate_proj.su": {"offset": 30771320, "shape": [576], "dtype": "f16"}, "model.layers.22.mlp.gate_proj.sv": {"offset": 30772472, "shape": [1536], "dtype": "f16"}, "model.layers.22.mlp.gate_proj.isc": {"offset": 30775544, "shape": [576], "dtype": "f16"}, "model.layers.22.mlp.gate_proj.osc": {"offset": 30776696, "shape": [1536], "dtype": "f16"}, "model.layers.22.mlp.gate_proj.scale": {"offset": 30779768, "shape": [1], "dtype": "f32"}, "model.layers.22.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.22.mlp.up_proj.bits": {"offset": 30779772, "shape": [3456, 24], "dtype": "u32"}, "model.layers.22.mlp.up_proj.su": {"offset": 31111548, "shape": [576], "dtype": "f16"}, "model.layers.22.mlp.up_proj.sv": {"offset": 31112700, "shape": [1536], "dtype": "f16"}, "model.layers.22.mlp.up_proj.isc": {"offset": 31115772, "shape": [576], "dtype": "f16"}, "model.layers.22.mlp.up_proj.osc": {"offset": 31116924, "shape": [1536], "dtype": "f16"}, "model.layers.22.mlp.up_proj.scale": {"offset": 31119996, "shape": [1], "dtype": "f32"}, "model.layers.22.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.22.mlp.down_proj.bits": {"offset": 31120000, "shape": [3456, 24], "dtype": "u32"}, "model.layers.22.mlp.down_proj.su": {"offset": 31451776, "shape": [1536], "dtype": "f16"}, "model.layers.22.mlp.down_proj.sv": {"offset": 31454848, "shape": [576], "dtype": "f16"}, "model.layers.22.mlp.down_proj.isc": {"offset": 31456000, "shape": [1536], "dtype": "f16"}, "model.layers.22.mlp.down_proj.osc": {"offset": 31459072, "shape": [576], "dtype": "f16"}, "model.layers.22.mlp.down_proj.scale": {"offset": 31460224, "shape": [1], "dtype": "f32"}, "model.layers.22.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.23.self_attn.q_proj.bits": {"offset": 31460228, "shape": [1296, 24], "dtype": "u32"}, "model.layers.23.self_attn.q_proj.su": {"offset": 31584644, "shape": [576], "dtype": "f16"}, "model.layers.23.self_attn.q_proj.sv": {"offset": 31585796, "shape": [576], "dtype": "f16"}, "model.layers.23.self_attn.q_proj.isc": {"offset": 31586948, "shape": [576], "dtype": "f16"}, "model.layers.23.self_attn.q_proj.osc": {"offset": 31588100, "shape": [576], "dtype": "f16"}, "model.layers.23.self_attn.q_proj.scale": {"offset": 31589252, "shape": [1], "dtype": "f32"}, "model.layers.23.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.23.self_attn.k_proj.bits": {"offset": 31589256, "shape": [432, 24], "dtype": "u32"}, "model.layers.23.self_attn.k_proj.su": {"offset": 31630728, "shape": [576], "dtype": "f16"}, "model.layers.23.self_attn.k_proj.sv": {"offset": 31631880, "shape": [192], "dtype": "f16"}, "model.layers.23.self_attn.k_proj.isc": {"offset": 31632264, "shape": [576], "dtype": "f16"}, "model.layers.23.self_attn.k_proj.osc": {"offset": 31633416, "shape": [192], "dtype": "f16"}, "model.layers.23.self_attn.k_proj.scale": {"offset": 31633800, "shape": [1], "dtype": "f32"}, "model.layers.23.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.23.self_attn.v_proj.bits": {"offset": 31633804, "shape": [432, 24], "dtype": "u32"}, "model.layers.23.self_attn.v_proj.su": {"offset": 31675276, "shape": [576], "dtype": "f16"}, "model.layers.23.self_attn.v_proj.sv": {"offset": 31676428, "shape": [192], "dtype": "f16"}, "model.layers.23.self_attn.v_proj.isc": {"offset": 31676812, "shape": [576], "dtype": "f16"}, "model.layers.23.self_attn.v_proj.osc": {"offset": 31677964, "shape": [192], "dtype": "f16"}, "model.layers.23.self_attn.v_proj.scale": {"offset": 31678348, "shape": [1], "dtype": "f32"}, "model.layers.23.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.23.self_attn.o_proj.bits": {"offset": 31678352, "shape": [1296, 24], "dtype": "u32"}, "model.layers.23.self_attn.o_proj.su": {"offset": 31802768, "shape": [576], "dtype": "f16"}, "model.layers.23.self_attn.o_proj.sv": {"offset": 31803920, "shape": [576], "dtype": "f16"}, "model.layers.23.self_attn.o_proj.isc": {"offset": 31805072, "shape": [576], "dtype": "f16"}, "model.layers.23.self_attn.o_proj.osc": {"offset": 31806224, "shape": [576], "dtype": "f16"}, "model.layers.23.self_attn.o_proj.scale": {"offset": 31807376, "shape": [1], "dtype": "f32"}, "model.layers.23.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.23.mlp.gate_proj.bits": {"offset": 31807380, "shape": [3456, 24], "dtype": "u32"}, "model.layers.23.mlp.gate_proj.su": {"offset": 32139156, "shape": [576], "dtype": "f16"}, "model.layers.23.mlp.gate_proj.sv": {"offset": 32140308, "shape": [1536], "dtype": "f16"}, "model.layers.23.mlp.gate_proj.isc": {"offset": 32143380, "shape": [576], "dtype": "f16"}, "model.layers.23.mlp.gate_proj.osc": {"offset": 32144532, "shape": [1536], "dtype": "f16"}, "model.layers.23.mlp.gate_proj.scale": {"offset": 32147604, "shape": [1], "dtype": "f32"}, "model.layers.23.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.23.mlp.up_proj.bits": {"offset": 32147608, "shape": [3456, 24], "dtype": "u32"}, "model.layers.23.mlp.up_proj.su": {"offset": 32479384, "shape": [576], "dtype": "f16"}, "model.layers.23.mlp.up_proj.sv": {"offset": 32480536, "shape": [1536], "dtype": "f16"}, "model.layers.23.mlp.up_proj.isc": {"offset": 32483608, "shape": [576], "dtype": "f16"}, "model.layers.23.mlp.up_proj.osc": {"offset": 32484760, "shape": [1536], "dtype": "f16"}, "model.layers.23.mlp.up_proj.scale": {"offset": 32487832, "shape": [1], "dtype": "f32"}, "model.layers.23.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.23.mlp.down_proj.bits": {"offset": 32487836, "shape": [3456, 24], "dtype": "u32"}, "model.layers.23.mlp.down_proj.su": {"offset": 32819612, "shape": [1536], "dtype": "f16"}, "model.layers.23.mlp.down_proj.sv": {"offset": 32822684, "shape": [576], "dtype": "f16"}, "model.layers.23.mlp.down_proj.isc": {"offset": 32823836, "shape": [1536], "dtype": "f16"}, "model.layers.23.mlp.down_proj.osc": {"offset": 32826908, "shape": [576], "dtype": "f16"}, "model.layers.23.mlp.down_proj.scale": {"offset": 32828060, "shape": [1], "dtype": "f32"}, "model.layers.23.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.24.self_attn.q_proj.bits": {"offset": 32828064, "shape": [1296, 24], "dtype": "u32"}, "model.layers.24.self_attn.q_proj.su": {"offset": 32952480, "shape": [576], "dtype": "f16"}, "model.layers.24.self_attn.q_proj.sv": {"offset": 32953632, "shape": [576], "dtype": "f16"}, "model.layers.24.self_attn.q_proj.isc": {"offset": 32954784, "shape": [576], "dtype": "f16"}, "model.layers.24.self_attn.q_proj.osc": {"offset": 32955936, "shape": [576], "dtype": "f16"}, "model.layers.24.self_attn.q_proj.scale": {"offset": 32957088, "shape": [1], "dtype": "f32"}, "model.layers.24.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.24.self_attn.k_proj.bits": {"offset": 32957092, "shape": [432, 24], "dtype": "u32"}, "model.layers.24.self_attn.k_proj.su": {"offset": 32998564, "shape": [576], "dtype": "f16"}, "model.layers.24.self_attn.k_proj.sv": {"offset": 32999716, "shape": [192], "dtype": "f16"}, "model.layers.24.self_attn.k_proj.isc": {"offset": 33000100, "shape": [576], "dtype": "f16"}, "model.layers.24.self_attn.k_proj.osc": {"offset": 33001252, "shape": [192], "dtype": "f16"}, "model.layers.24.self_attn.k_proj.scale": {"offset": 33001636, "shape": [1], "dtype": "f32"}, "model.layers.24.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.24.self_attn.v_proj.bits": {"offset": 33001640, "shape": [432, 24], "dtype": "u32"}, "model.layers.24.self_attn.v_proj.su": {"offset": 33043112, "shape": [576], "dtype": "f16"}, "model.layers.24.self_attn.v_proj.sv": {"offset": 33044264, "shape": [192], "dtype": "f16"}, "model.layers.24.self_attn.v_proj.isc": {"offset": 33044648, "shape": [576], "dtype": "f16"}, "model.layers.24.self_attn.v_proj.osc": {"offset": 33045800, "shape": [192], "dtype": "f16"}, "model.layers.24.self_attn.v_proj.scale": {"offset": 33046184, "shape": [1], "dtype": "f32"}, "model.layers.24.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.24.self_attn.o_proj.bits": {"offset": 33046188, "shape": [1296, 24], "dtype": "u32"}, "model.layers.24.self_attn.o_proj.su": {"offset": 33170604, "shape": [576], "dtype": "f16"}, "model.layers.24.self_attn.o_proj.sv": {"offset": 33171756, "shape": [576], "dtype": "f16"}, "model.layers.24.self_attn.o_proj.isc": {"offset": 33172908, "shape": [576], "dtype": "f16"}, "model.layers.24.self_attn.o_proj.osc": {"offset": 33174060, "shape": [576], "dtype": "f16"}, "model.layers.24.self_attn.o_proj.scale": {"offset": 33175212, "shape": [1], "dtype": "f32"}, "model.layers.24.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.24.mlp.gate_proj.bits": {"offset": 33175216, "shape": [3456, 24], "dtype": "u32"}, "model.layers.24.mlp.gate_proj.su": {"offset": 33506992, "shape": [576], "dtype": "f16"}, "model.layers.24.mlp.gate_proj.sv": {"offset": 33508144, "shape": [1536], "dtype": "f16"}, "model.layers.24.mlp.gate_proj.isc": {"offset": 33511216, "shape": [576], "dtype": "f16"}, "model.layers.24.mlp.gate_proj.osc": {"offset": 33512368, "shape": [1536], "dtype": "f16"}, "model.layers.24.mlp.gate_proj.scale": {"offset": 33515440, "shape": [1], "dtype": "f32"}, "model.layers.24.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.24.mlp.up_proj.bits": {"offset": 33515444, "shape": [3456, 24], "dtype": "u32"}, "model.layers.24.mlp.up_proj.su": {"offset": 33847220, "shape": [576], "dtype": "f16"}, "model.layers.24.mlp.up_proj.sv": {"offset": 33848372, "shape": [1536], "dtype": "f16"}, "model.layers.24.mlp.up_proj.isc": {"offset": 33851444, "shape": [576], "dtype": "f16"}, "model.layers.24.mlp.up_proj.osc": {"offset": 33852596, "shape": [1536], "dtype": "f16"}, "model.layers.24.mlp.up_proj.scale": {"offset": 33855668, "shape": [1], "dtype": "f32"}, "model.layers.24.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.24.mlp.down_proj.bits": {"offset": 33855672, "shape": [3456, 24], "dtype": "u32"}, "model.layers.24.mlp.down_proj.su": {"offset": 34187448, "shape": [1536], "dtype": "f16"}, "model.layers.24.mlp.down_proj.sv": {"offset": 34190520, "shape": [576], "dtype": "f16"}, "model.layers.24.mlp.down_proj.isc": {"offset": 34191672, "shape": [1536], "dtype": "f16"}, "model.layers.24.mlp.down_proj.osc": {"offset": 34194744, "shape": [576], "dtype": "f16"}, "model.layers.24.mlp.down_proj.scale": {"offset": 34195896, "shape": [1], "dtype": "f32"}, "model.layers.24.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.25.self_attn.q_proj.bits": {"offset": 34195900, "shape": [1296, 24], "dtype": "u32"}, "model.layers.25.self_attn.q_proj.su": {"offset": 34320316, "shape": [576], "dtype": "f16"}, "model.layers.25.self_attn.q_proj.sv": {"offset": 34321468, "shape": [576], "dtype": "f16"}, "model.layers.25.self_attn.q_proj.isc": {"offset": 34322620, "shape": [576], "dtype": "f16"}, "model.layers.25.self_attn.q_proj.osc": {"offset": 34323772, "shape": [576], "dtype": "f16"}, "model.layers.25.self_attn.q_proj.scale": {"offset": 34324924, "shape": [1], "dtype": "f32"}, "model.layers.25.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.25.self_attn.k_proj.bits": {"offset": 34324928, "shape": [432, 24], "dtype": "u32"}, "model.layers.25.self_attn.k_proj.su": {"offset": 34366400, "shape": [576], "dtype": "f16"}, "model.layers.25.self_attn.k_proj.sv": {"offset": 34367552, "shape": [192], "dtype": "f16"}, "model.layers.25.self_attn.k_proj.isc": {"offset": 34367936, "shape": [576], "dtype": "f16"}, "model.layers.25.self_attn.k_proj.osc": {"offset": 34369088, "shape": [192], "dtype": "f16"}, "model.layers.25.self_attn.k_proj.scale": {"offset": 34369472, "shape": [1], "dtype": "f32"}, "model.layers.25.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.25.self_attn.v_proj.bits": {"offset": 34369476, "shape": [432, 24], "dtype": "u32"}, "model.layers.25.self_attn.v_proj.su": {"offset": 34410948, "shape": [576], "dtype": "f16"}, "model.layers.25.self_attn.v_proj.sv": {"offset": 34412100, "shape": [192], "dtype": "f16"}, "model.layers.25.self_attn.v_proj.isc": {"offset": 34412484, "shape": [576], "dtype": "f16"}, "model.layers.25.self_attn.v_proj.osc": {"offset": 34413636, "shape": [192], "dtype": "f16"}, "model.layers.25.self_attn.v_proj.scale": {"offset": 34414020, "shape": [1], "dtype": "f32"}, "model.layers.25.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.25.self_attn.o_proj.bits": {"offset": 34414024, "shape": [1296, 24], "dtype": "u32"}, "model.layers.25.self_attn.o_proj.su": {"offset": 34538440, "shape": [576], "dtype": "f16"}, "model.layers.25.self_attn.o_proj.sv": {"offset": 34539592, "shape": [576], "dtype": "f16"}, "model.layers.25.self_attn.o_proj.isc": {"offset": 34540744, "shape": [576], "dtype": "f16"}, "model.layers.25.self_attn.o_proj.osc": {"offset": 34541896, "shape": [576], "dtype": "f16"}, "model.layers.25.self_attn.o_proj.scale": {"offset": 34543048, "shape": [1], "dtype": "f32"}, "model.layers.25.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.25.mlp.gate_proj.bits": {"offset": 34543052, "shape": [3456, 24], "dtype": "u32"}, "model.layers.25.mlp.gate_proj.su": {"offset": 34874828, "shape": [576], "dtype": "f16"}, "model.layers.25.mlp.gate_proj.sv": {"offset": 34875980, "shape": [1536], "dtype": "f16"}, "model.layers.25.mlp.gate_proj.isc": {"offset": 34879052, "shape": [576], "dtype": "f16"}, "model.layers.25.mlp.gate_proj.osc": {"offset": 34880204, "shape": [1536], "dtype": "f16"}, "model.layers.25.mlp.gate_proj.scale": {"offset": 34883276, "shape": [1], "dtype": "f32"}, "model.layers.25.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.25.mlp.up_proj.bits": {"offset": 34883280, "shape": [3456, 24], "dtype": "u32"}, "model.layers.25.mlp.up_proj.su": {"offset": 35215056, "shape": [576], "dtype": "f16"}, "model.layers.25.mlp.up_proj.sv": {"offset": 35216208, "shape": [1536], "dtype": "f16"}, "model.layers.25.mlp.up_proj.isc": {"offset": 35219280, "shape": [576], "dtype": "f16"}, "model.layers.25.mlp.up_proj.osc": {"offset": 35220432, "shape": [1536], "dtype": "f16"}, "model.layers.25.mlp.up_proj.scale": {"offset": 35223504, "shape": [1], "dtype": "f32"}, "model.layers.25.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.25.mlp.down_proj.bits": {"offset": 35223508, "shape": [3456, 24], "dtype": "u32"}, "model.layers.25.mlp.down_proj.su": {"offset": 35555284, "shape": [1536], "dtype": "f16"}, "model.layers.25.mlp.down_proj.sv": {"offset": 35558356, "shape": [576], "dtype": "f16"}, "model.layers.25.mlp.down_proj.isc": {"offset": 35559508, "shape": [1536], "dtype": "f16"}, "model.layers.25.mlp.down_proj.osc": {"offset": 35562580, "shape": [576], "dtype": "f16"}, "model.layers.25.mlp.down_proj.scale": {"offset": 35563732, "shape": [1], "dtype": "f32"}, "model.layers.25.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.26.self_attn.q_proj.bits": {"offset": 35563736, "shape": [1296, 24], "dtype": "u32"}, "model.layers.26.self_attn.q_proj.su": {"offset": 35688152, "shape": [576], "dtype": "f16"}, "model.layers.26.self_attn.q_proj.sv": {"offset": 35689304, "shape": [576], "dtype": "f16"}, "model.layers.26.self_attn.q_proj.isc": {"offset": 35690456, "shape": [576], "dtype": "f16"}, "model.layers.26.self_attn.q_proj.osc": {"offset": 35691608, "shape": [576], "dtype": "f16"}, "model.layers.26.self_attn.q_proj.scale": {"offset": 35692760, "shape": [1], "dtype": "f32"}, "model.layers.26.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.26.self_attn.k_proj.bits": {"offset": 35692764, "shape": [432, 24], "dtype": "u32"}, "model.layers.26.self_attn.k_proj.su": {"offset": 35734236, "shape": [576], "dtype": "f16"}, "model.layers.26.self_attn.k_proj.sv": {"offset": 35735388, "shape": [192], "dtype": "f16"}, "model.layers.26.self_attn.k_proj.isc": {"offset": 35735772, "shape": [576], "dtype": "f16"}, "model.layers.26.self_attn.k_proj.osc": {"offset": 35736924, "shape": [192], "dtype": "f16"}, "model.layers.26.self_attn.k_proj.scale": {"offset": 35737308, "shape": [1], "dtype": "f32"}, "model.layers.26.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.26.self_attn.v_proj.bits": {"offset": 35737312, "shape": [432, 24], "dtype": "u32"}, "model.layers.26.self_attn.v_proj.su": {"offset": 35778784, "shape": [576], "dtype": "f16"}, "model.layers.26.self_attn.v_proj.sv": {"offset": 35779936, "shape": [192], "dtype": "f16"}, "model.layers.26.self_attn.v_proj.isc": {"offset": 35780320, "shape": [576], "dtype": "f16"}, "model.layers.26.self_attn.v_proj.osc": {"offset": 35781472, "shape": [192], "dtype": "f16"}, "model.layers.26.self_attn.v_proj.scale": {"offset": 35781856, "shape": [1], "dtype": "f32"}, "model.layers.26.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.26.self_attn.o_proj.bits": {"offset": 35781860, "shape": [1296, 24], "dtype": "u32"}, "model.layers.26.self_attn.o_proj.su": {"offset": 35906276, "shape": [576], "dtype": "f16"}, "model.layers.26.self_attn.o_proj.sv": {"offset": 35907428, "shape": [576], "dtype": "f16"}, "model.layers.26.self_attn.o_proj.isc": {"offset": 35908580, "shape": [576], "dtype": "f16"}, "model.layers.26.self_attn.o_proj.osc": {"offset": 35909732, "shape": [576], "dtype": "f16"}, "model.layers.26.self_attn.o_proj.scale": {"offset": 35910884, "shape": [1], "dtype": "f32"}, "model.layers.26.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.26.mlp.gate_proj.bits": {"offset": 35910888, "shape": [3456, 24], "dtype": "u32"}, "model.layers.26.mlp.gate_proj.su": {"offset": 36242664, "shape": [576], "dtype": "f16"}, "model.layers.26.mlp.gate_proj.sv": {"offset": 36243816, "shape": [1536], "dtype": "f16"}, "model.layers.26.mlp.gate_proj.isc": {"offset": 36246888, "shape": [576], "dtype": "f16"}, "model.layers.26.mlp.gate_proj.osc": {"offset": 36248040, "shape": [1536], "dtype": "f16"}, "model.layers.26.mlp.gate_proj.scale": {"offset": 36251112, "shape": [1], "dtype": "f32"}, "model.layers.26.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.26.mlp.up_proj.bits": {"offset": 36251116, "shape": [3456, 24], "dtype": "u32"}, "model.layers.26.mlp.up_proj.su": {"offset": 36582892, "shape": [576], "dtype": "f16"}, "model.layers.26.mlp.up_proj.sv": {"offset": 36584044, "shape": [1536], "dtype": "f16"}, "model.layers.26.mlp.up_proj.isc": {"offset": 36587116, "shape": [576], "dtype": "f16"}, "model.layers.26.mlp.up_proj.osc": {"offset": 36588268, "shape": [1536], "dtype": "f16"}, "model.layers.26.mlp.up_proj.scale": {"offset": 36591340, "shape": [1], "dtype": "f32"}, "model.layers.26.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.26.mlp.down_proj.bits": {"offset": 36591344, "shape": [3456, 24], "dtype": "u32"}, "model.layers.26.mlp.down_proj.su": {"offset": 36923120, "shape": [1536], "dtype": "f16"}, "model.layers.26.mlp.down_proj.sv": {"offset": 36926192, "shape": [576], "dtype": "f16"}, "model.layers.26.mlp.down_proj.isc": {"offset": 36927344, "shape": [1536], "dtype": "f16"}, "model.layers.26.mlp.down_proj.osc": {"offset": 36930416, "shape": [576], "dtype": "f16"}, "model.layers.26.mlp.down_proj.scale": {"offset": 36931568, "shape": [1], "dtype": "f32"}, "model.layers.26.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.27.self_attn.q_proj.bits": {"offset": 36931572, "shape": [1296, 24], "dtype": "u32"}, "model.layers.27.self_attn.q_proj.su": {"offset": 37055988, "shape": [576], "dtype": "f16"}, "model.layers.27.self_attn.q_proj.sv": {"offset": 37057140, "shape": [576], "dtype": "f16"}, "model.layers.27.self_attn.q_proj.isc": {"offset": 37058292, "shape": [576], "dtype": "f16"}, "model.layers.27.self_attn.q_proj.osc": {"offset": 37059444, "shape": [576], "dtype": "f16"}, "model.layers.27.self_attn.q_proj.scale": {"offset": 37060596, "shape": [1], "dtype": "f32"}, "model.layers.27.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.27.self_attn.k_proj.bits": {"offset": 37060600, "shape": [432, 24], "dtype": "u32"}, "model.layers.27.self_attn.k_proj.su": {"offset": 37102072, "shape": [576], "dtype": "f16"}, "model.layers.27.self_attn.k_proj.sv": {"offset": 37103224, "shape": [192], "dtype": "f16"}, "model.layers.27.self_attn.k_proj.isc": {"offset": 37103608, "shape": [576], "dtype": "f16"}, "model.layers.27.self_attn.k_proj.osc": {"offset": 37104760, "shape": [192], "dtype": "f16"}, "model.layers.27.self_attn.k_proj.scale": {"offset": 37105144, "shape": [1], "dtype": "f32"}, "model.layers.27.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.27.self_attn.v_proj.bits": {"offset": 37105148, "shape": [432, 24], "dtype": "u32"}, "model.layers.27.self_attn.v_proj.su": {"offset": 37146620, "shape": [576], "dtype": "f16"}, "model.layers.27.self_attn.v_proj.sv": {"offset": 37147772, "shape": [192], "dtype": "f16"}, "model.layers.27.self_attn.v_proj.isc": {"offset": 37148156, "shape": [576], "dtype": "f16"}, "model.layers.27.self_attn.v_proj.osc": {"offset": 37149308, "shape": [192], "dtype": "f16"}, "model.layers.27.self_attn.v_proj.scale": {"offset": 37149692, "shape": [1], "dtype": "f32"}, "model.layers.27.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.27.self_attn.o_proj.bits": {"offset": 37149696, "shape": [1296, 24], "dtype": "u32"}, "model.layers.27.self_attn.o_proj.su": {"offset": 37274112, "shape": [576], "dtype": "f16"}, "model.layers.27.self_attn.o_proj.sv": {"offset": 37275264, "shape": [576], "dtype": "f16"}, "model.layers.27.self_attn.o_proj.isc": {"offset": 37276416, "shape": [576], "dtype": "f16"}, "model.layers.27.self_attn.o_proj.osc": {"offset": 37277568, "shape": [576], "dtype": "f16"}, "model.layers.27.self_attn.o_proj.scale": {"offset": 37278720, "shape": [1], "dtype": "f32"}, "model.layers.27.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.27.mlp.gate_proj.bits": {"offset": 37278724, "shape": [3456, 24], "dtype": "u32"}, "model.layers.27.mlp.gate_proj.su": {"offset": 37610500, "shape": [576], "dtype": "f16"}, "model.layers.27.mlp.gate_proj.sv": {"offset": 37611652, "shape": [1536], "dtype": "f16"}, "model.layers.27.mlp.gate_proj.isc": {"offset": 37614724, "shape": [576], "dtype": "f16"}, "model.layers.27.mlp.gate_proj.osc": {"offset": 37615876, "shape": [1536], "dtype": "f16"}, "model.layers.27.mlp.gate_proj.scale": {"offset": 37618948, "shape": [1], "dtype": "f32"}, "model.layers.27.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.27.mlp.up_proj.bits": {"offset": 37618952, "shape": [3456, 24], "dtype": "u32"}, "model.layers.27.mlp.up_proj.su": {"offset": 37950728, "shape": [576], "dtype": "f16"}, "model.layers.27.mlp.up_proj.sv": {"offset": 37951880, "shape": [1536], "dtype": "f16"}, "model.layers.27.mlp.up_proj.isc": {"offset": 37954952, "shape": [576], "dtype": "f16"}, "model.layers.27.mlp.up_proj.osc": {"offset": 37956104, "shape": [1536], "dtype": "f16"}, "model.layers.27.mlp.up_proj.scale": {"offset": 37959176, "shape": [1], "dtype": "f32"}, "model.layers.27.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.27.mlp.down_proj.bits": {"offset": 37959180, "shape": [3456, 24], "dtype": "u32"}, "model.layers.27.mlp.down_proj.su": {"offset": 38290956, "shape": [1536], "dtype": "f16"}, "model.layers.27.mlp.down_proj.sv": {"offset": 38294028, "shape": [576], "dtype": "f16"}, "model.layers.27.mlp.down_proj.isc": {"offset": 38295180, "shape": [1536], "dtype": "f16"}, "model.layers.27.mlp.down_proj.osc": {"offset": 38298252, "shape": [576], "dtype": "f16"}, "model.layers.27.mlp.down_proj.scale": {"offset": 38299404, "shape": [1], "dtype": "f32"}, "model.layers.27.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.28.self_attn.q_proj.bits": {"offset": 38299408, "shape": [1296, 24], "dtype": "u32"}, "model.layers.28.self_attn.q_proj.su": {"offset": 38423824, "shape": [576], "dtype": "f16"}, "model.layers.28.self_attn.q_proj.sv": {"offset": 38424976, "shape": [576], "dtype": "f16"}, "model.layers.28.self_attn.q_proj.isc": {"offset": 38426128, "shape": [576], "dtype": "f16"}, "model.layers.28.self_attn.q_proj.osc": {"offset": 38427280, "shape": [576], "dtype": "f16"}, "model.layers.28.self_attn.q_proj.scale": {"offset": 38428432, "shape": [1], "dtype": "f32"}, "model.layers.28.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.28.self_attn.k_proj.bits": {"offset": 38428436, "shape": [432, 24], "dtype": "u32"}, "model.layers.28.self_attn.k_proj.su": {"offset": 38469908, "shape": [576], "dtype": "f16"}, "model.layers.28.self_attn.k_proj.sv": {"offset": 38471060, "shape": [192], "dtype": "f16"}, "model.layers.28.self_attn.k_proj.isc": {"offset": 38471444, "shape": [576], "dtype": "f16"}, "model.layers.28.self_attn.k_proj.osc": {"offset": 38472596, "shape": [192], "dtype": "f16"}, "model.layers.28.self_attn.k_proj.scale": {"offset": 38472980, "shape": [1], "dtype": "f32"}, "model.layers.28.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.28.self_attn.v_proj.bits": {"offset": 38472984, "shape": [432, 24], "dtype": "u32"}, "model.layers.28.self_attn.v_proj.su": {"offset": 38514456, "shape": [576], "dtype": "f16"}, "model.layers.28.self_attn.v_proj.sv": {"offset": 38515608, "shape": [192], "dtype": "f16"}, "model.layers.28.self_attn.v_proj.isc": {"offset": 38515992, "shape": [576], "dtype": "f16"}, "model.layers.28.self_attn.v_proj.osc": {"offset": 38517144, "shape": [192], "dtype": "f16"}, "model.layers.28.self_attn.v_proj.scale": {"offset": 38517528, "shape": [1], "dtype": "f32"}, "model.layers.28.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.28.self_attn.o_proj.bits": {"offset": 38517532, "shape": [1296, 24], "dtype": "u32"}, "model.layers.28.self_attn.o_proj.su": {"offset": 38641948, "shape": [576], "dtype": "f16"}, "model.layers.28.self_attn.o_proj.sv": {"offset": 38643100, "shape": [576], "dtype": "f16"}, "model.layers.28.self_attn.o_proj.isc": {"offset": 38644252, "shape": [576], "dtype": "f16"}, "model.layers.28.self_attn.o_proj.osc": {"offset": 38645404, "shape": [576], "dtype": "f16"}, "model.layers.28.self_attn.o_proj.scale": {"offset": 38646556, "shape": [1], "dtype": "f32"}, "model.layers.28.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.28.mlp.gate_proj.bits": {"offset": 38646560, "shape": [3456, 24], "dtype": "u32"}, "model.layers.28.mlp.gate_proj.su": {"offset": 38978336, "shape": [576], "dtype": "f16"}, "model.layers.28.mlp.gate_proj.sv": {"offset": 38979488, "shape": [1536], "dtype": "f16"}, "model.layers.28.mlp.gate_proj.isc": {"offset": 38982560, "shape": [576], "dtype": "f16"}, "model.layers.28.mlp.gate_proj.osc": {"offset": 38983712, "shape": [1536], "dtype": "f16"}, "model.layers.28.mlp.gate_proj.scale": {"offset": 38986784, "shape": [1], "dtype": "f32"}, "model.layers.28.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.28.mlp.up_proj.bits": {"offset": 38986788, "shape": [3456, 24], "dtype": "u32"}, "model.layers.28.mlp.up_proj.su": {"offset": 39318564, "shape": [576], "dtype": "f16"}, "model.layers.28.mlp.up_proj.sv": {"offset": 39319716, "shape": [1536], "dtype": "f16"}, "model.layers.28.mlp.up_proj.isc": {"offset": 39322788, "shape": [576], "dtype": "f16"}, "model.layers.28.mlp.up_proj.osc": {"offset": 39323940, "shape": [1536], "dtype": "f16"}, "model.layers.28.mlp.up_proj.scale": {"offset": 39327012, "shape": [1], "dtype": "f32"}, "model.layers.28.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.28.mlp.down_proj.bits": {"offset": 39327016, "shape": [3456, 24], "dtype": "u32"}, "model.layers.28.mlp.down_proj.su": {"offset": 39658792, "shape": [1536], "dtype": "f16"}, "model.layers.28.mlp.down_proj.sv": {"offset": 39661864, "shape": [576], "dtype": "f16"}, "model.layers.28.mlp.down_proj.isc": {"offset": 39663016, "shape": [1536], "dtype": "f16"}, "model.layers.28.mlp.down_proj.osc": {"offset": 39666088, "shape": [576], "dtype": "f16"}, "model.layers.28.mlp.down_proj.scale": {"offset": 39667240, "shape": [1], "dtype": "f32"}, "model.layers.28.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.layers.29.self_attn.q_proj.bits": {"offset": 39667244, "shape": [1296, 24], "dtype": "u32"}, "model.layers.29.self_attn.q_proj.su": {"offset": 39791660, "shape": [576], "dtype": "f16"}, "model.layers.29.self_attn.q_proj.sv": {"offset": 39792812, "shape": [576], "dtype": "f16"}, "model.layers.29.self_attn.q_proj.isc": {"offset": 39793964, "shape": [576], "dtype": "f16"}, "model.layers.29.self_attn.q_proj.osc": {"offset": 39795116, "shape": [576], "dtype": "f16"}, "model.layers.29.self_attn.q_proj.scale": {"offset": 39796268, "shape": [1], "dtype": "f32"}, "model.layers.29.self_attn.q_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.29.self_attn.k_proj.bits": {"offset": 39796272, "shape": [432, 24], "dtype": "u32"}, "model.layers.29.self_attn.k_proj.su": {"offset": 39837744, "shape": [576], "dtype": "f16"}, "model.layers.29.self_attn.k_proj.sv": {"offset": 39838896, "shape": [192], "dtype": "f16"}, "model.layers.29.self_attn.k_proj.isc": {"offset": 39839280, "shape": [576], "dtype": "f16"}, "model.layers.29.self_attn.k_proj.osc": {"offset": 39840432, "shape": [192], "dtype": "f16"}, "model.layers.29.self_attn.k_proj.scale": {"offset": 39840816, "shape": [1], "dtype": "f32"}, "model.layers.29.self_attn.k_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.29.self_attn.v_proj.bits": {"offset": 39840820, "shape": [432, 24], "dtype": "u32"}, "model.layers.29.self_attn.v_proj.su": {"offset": 39882292, "shape": [576], "dtype": "f16"}, "model.layers.29.self_attn.v_proj.sv": {"offset": 39883444, "shape": [192], "dtype": "f16"}, "model.layers.29.self_attn.v_proj.isc": {"offset": 39883828, "shape": [576], "dtype": "f16"}, "model.layers.29.self_attn.v_proj.osc": {"offset": 39884980, "shape": [192], "dtype": "f16"}, "model.layers.29.self_attn.v_proj.scale": {"offset": 39885364, "shape": [1], "dtype": "f32"}, "model.layers.29.self_attn.v_proj": {"quant": true, "n_out": 192, "k_in": 576, "K": 3, "block": 64, "tiles_out": 12, "tiles_in": 36}, "model.layers.29.self_attn.o_proj.bits": {"offset": 39885368, "shape": [1296, 24], "dtype": "u32"}, "model.layers.29.self_attn.o_proj.su": {"offset": 40009784, "shape": [576], "dtype": "f16"}, "model.layers.29.self_attn.o_proj.sv": {"offset": 40010936, "shape": [576], "dtype": "f16"}, "model.layers.29.self_attn.o_proj.isc": {"offset": 40012088, "shape": [576], "dtype": "f16"}, "model.layers.29.self_attn.o_proj.osc": {"offset": 40013240, "shape": [576], "dtype": "f16"}, "model.layers.29.self_attn.o_proj.scale": {"offset": 40014392, "shape": [1], "dtype": "f32"}, "model.layers.29.self_attn.o_proj": {"quant": true, "n_out": 576, "k_in": 576, "K": 3, "block": 64, "tiles_out": 36, "tiles_in": 36}, "model.layers.29.mlp.gate_proj.bits": {"offset": 40014396, "shape": [3456, 24], "dtype": "u32"}, "model.layers.29.mlp.gate_proj.su": {"offset": 40346172, "shape": [576], "dtype": "f16"}, "model.layers.29.mlp.gate_proj.sv": {"offset": 40347324, "shape": [1536], "dtype": "f16"}, "model.layers.29.mlp.gate_proj.isc": {"offset": 40350396, "shape": [576], "dtype": "f16"}, "model.layers.29.mlp.gate_proj.osc": {"offset": 40351548, "shape": [1536], "dtype": "f16"}, "model.layers.29.mlp.gate_proj.scale": {"offset": 40354620, "shape": [1], "dtype": "f32"}, "model.layers.29.mlp.gate_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.29.mlp.up_proj.bits": {"offset": 40354624, "shape": [3456, 24], "dtype": "u32"}, "model.layers.29.mlp.up_proj.su": {"offset": 40686400, "shape": [576], "dtype": "f16"}, "model.layers.29.mlp.up_proj.sv": {"offset": 40687552, "shape": [1536], "dtype": "f16"}, "model.layers.29.mlp.up_proj.isc": {"offset": 40690624, "shape": [576], "dtype": "f16"}, "model.layers.29.mlp.up_proj.osc": {"offset": 40691776, "shape": [1536], "dtype": "f16"}, "model.layers.29.mlp.up_proj.scale": {"offset": 40694848, "shape": [1], "dtype": "f32"}, "model.layers.29.mlp.up_proj": {"quant": true, "n_out": 1536, "k_in": 576, "K": 3, "block": 64, "tiles_out": 96, "tiles_in": 36}, "model.layers.29.mlp.down_proj.bits": {"offset": 40694852, "shape": [3456, 24], "dtype": "u32"}, "model.layers.29.mlp.down_proj.su": {"offset": 41026628, "shape": [1536], "dtype": "f16"}, "model.layers.29.mlp.down_proj.sv": {"offset": 41029700, "shape": [576], "dtype": "f16"}, "model.layers.29.mlp.down_proj.isc": {"offset": 41030852, "shape": [1536], "dtype": "f16"}, "model.layers.29.mlp.down_proj.osc": {"offset": 41033924, "shape": [576], "dtype": "f16"}, "model.layers.29.mlp.down_proj.scale": {"offset": 41035076, "shape": [1], "dtype": "f32"}, "model.layers.29.mlp.down_proj": {"quant": true, "n_out": 576, "k_in": 1536, "K": 3, "block": 128, "tiles_out": 36, "tiles_in": 96}, "model.embed_tokens.weight": {"offset": 41035080, "shape": [49152, 576], "dtype": "f16"}, "model.layers.0.input_layernorm.weight": {"offset": 97658184, "shape": [576], "dtype": "f16"}, "model.layers.0.post_attention_layernorm.weight": {"offset": 97659336, "shape": [576], "dtype": "f16"}, "model.layers.1.input_layernorm.weight": {"offset": 97660488, "shape": [576], "dtype": "f16"}, "model.layers.1.post_attention_layernorm.weight": {"offset": 97661640, "shape": [576], "dtype": "f16"}, "model.layers.2.input_layernorm.weight": {"offset": 97662792, "shape": [576], "dtype": "f16"}, "model.layers.2.post_attention_layernorm.weight": {"offset": 97663944, "shape": [576], "dtype": "f16"}, "model.layers.3.input_layernorm.weight": {"offset": 97665096, "shape": [576], "dtype": "f16"}, "model.layers.3.post_attention_layernorm.weight": {"offset": 97666248, "shape": [576], "dtype": "f16"}, "model.layers.4.input_layernorm.weight": {"offset": 97667400, "shape": [576], "dtype": "f16"}, "model.layers.4.post_attention_layernorm.weight": {"offset": 97668552, "shape": [576], "dtype": "f16"}, "model.layers.5.input_layernorm.weight": {"offset": 97669704, "shape": [576], "dtype": "f16"}, "model.layers.5.post_attention_layernorm.weight": {"offset": 97670856, "shape": [576], "dtype": "f16"}, "model.layers.6.input_layernorm.weight": {"offset": 97672008, "shape": [576], "dtype": "f16"}, "model.layers.6.post_attention_layernorm.weight": {"offset": 97673160, "shape": [576], "dtype": "f16"}, "model.layers.7.input_layernorm.weight": {"offset": 97674312, "shape": [576], "dtype": "f16"}, "model.layers.7.post_attention_layernorm.weight": {"offset": 97675464, "shape": [576], "dtype": "f16"}, "model.layers.8.input_layernorm.weight": {"offset": 97676616, "shape": [576], "dtype": "f16"}, "model.layers.8.post_attention_layernorm.weight": {"offset": 97677768, "shape": [576], "dtype": "f16"}, "model.layers.9.input_layernorm.weight": {"offset": 97678920, "shape": [576], "dtype": "f16"}, "model.layers.9.post_attention_layernorm.weight": {"offset": 97680072, "shape": [576], "dtype": "f16"}, "model.layers.10.input_layernorm.weight": {"offset": 97681224, "shape": [576], "dtype": "f16"}, "model.layers.10.post_attention_layernorm.weight": {"offset": 97682376, "shape": [576], "dtype": "f16"}, "model.layers.11.input_layernorm.weight": {"offset": 97683528, "shape": [576], "dtype": "f16"}, "model.layers.11.post_attention_layernorm.weight": {"offset": 97684680, "shape": [576], "dtype": "f16"}, "model.layers.12.input_layernorm.weight": {"offset": 97685832, "shape": [576], "dtype": "f16"}, "model.layers.12.post_attention_layernorm.weight": {"offset": 97686984, "shape": [576], "dtype": "f16"}, "model.layers.13.input_layernorm.weight": {"offset": 97688136, "shape": [576], "dtype": "f16"}, "model.layers.13.post_attention_layernorm.weight": {"offset": 97689288, "shape": [576], "dtype": "f16"}, "model.layers.14.input_layernorm.weight": {"offset": 97690440, "shape": [576], "dtype": "f16"}, "model.layers.14.post_attention_layernorm.weight": {"offset": 97691592, "shape": [576], "dtype": "f16"}, "model.layers.15.input_layernorm.weight": {"offset": 97692744, "shape": [576], "dtype": "f16"}, "model.layers.15.post_attention_layernorm.weight": {"offset": 97693896, "shape": [576], "dtype": "f16"}, "model.layers.16.input_layernorm.weight": {"offset": 97695048, "shape": [576], "dtype": "f16"}, "model.layers.16.post_attention_layernorm.weight": {"offset": 97696200, "shape": [576], "dtype": "f16"}, "model.layers.17.input_layernorm.weight": {"offset": 97697352, "shape": [576], "dtype": "f16"}, "model.layers.17.post_attention_layernorm.weight": {"offset": 97698504, "shape": [576], "dtype": "f16"}, "model.layers.18.input_layernorm.weight": {"offset": 97699656, "shape": [576], "dtype": "f16"}, "model.layers.18.post_attention_layernorm.weight": {"offset": 97700808, "shape": [576], "dtype": "f16"}, "model.layers.19.input_layernorm.weight": {"offset": 97701960, "shape": [576], "dtype": "f16"}, "model.layers.19.post_attention_layernorm.weight": {"offset": 97703112, "shape": [576], "dtype": "f16"}, "model.layers.20.input_layernorm.weight": {"offset": 97704264, "shape": [576], "dtype": "f16"}, "model.layers.20.post_attention_layernorm.weight": {"offset": 97705416, "shape": [576], "dtype": "f16"}, "model.layers.21.input_layernorm.weight": {"offset": 97706568, "shape": [576], "dtype": "f16"}, "model.layers.21.post_attention_layernorm.weight": {"offset": 97707720, "shape": [576], "dtype": "f16"}, "model.layers.22.input_layernorm.weight": {"offset": 97708872, "shape": [576], "dtype": "f16"}, "model.layers.22.post_attention_layernorm.weight": {"offset": 97710024, "shape": [576], "dtype": "f16"}, "model.layers.23.input_layernorm.weight": {"offset": 97711176, "shape": [576], "dtype": "f16"}, "model.layers.23.post_attention_layernorm.weight": {"offset": 97712328, "shape": [576], "dtype": "f16"}, "model.layers.24.input_layernorm.weight": {"offset": 97713480, "shape": [576], "dtype": "f16"}, "model.layers.24.post_attention_layernorm.weight": {"offset": 97714632, "shape": [576], "dtype": "f16"}, "model.layers.25.input_layernorm.weight": {"offset": 97715784, "shape": [576], "dtype": "f16"}, "model.layers.25.post_attention_layernorm.weight": {"offset": 97716936, "shape": [576], "dtype": "f16"}, "model.layers.26.input_layernorm.weight": {"offset": 97718088, "shape": [576], "dtype": "f16"}, "model.layers.26.post_attention_layernorm.weight": {"offset": 97719240, "shape": [576], "dtype": "f16"}, "model.layers.27.input_layernorm.weight": {"offset": 97720392, "shape": [576], "dtype": "f16"}, "model.layers.27.post_attention_layernorm.weight": {"offset": 97721544, "shape": [576], "dtype": "f16"}, "model.layers.28.input_layernorm.weight": {"offset": 97722696, "shape": [576], "dtype": "f16"}, "model.layers.28.post_attention_layernorm.weight": {"offset": 97723848, "shape": [576], "dtype": "f16"}, "model.layers.29.input_layernorm.weight": {"offset": 97725000, "shape": [576], "dtype": "f16"}, "model.layers.29.post_attention_layernorm.weight": {"offset": 97726152, "shape": [576], "dtype": "f16"}, "model.norm.weight": {"offset": 97727304, "shape": [576], "dtype": "f16"}, "lm_head.weight": {"offset": 97728456, "shape": [49152, 576], "dtype": "f16"}}