{ "component": "refiner", "cuda_peak_allocated_bytes": 2239758336, "cuda_peak_reserved_bytes": 2277507072, "generic_modules": { "actual_group_size_counts": { "-1": 1, "64": 338 }, "count": 339, "entries": [ { "group_size": -1, "logical_numel": 131072, "logical_shape": [ 2048, 64 ], "module": "patch_embedder", "original_bytes": 262144, "parameter": "patch_embedder.weight", "re_quantize_for_matmul": false, "stored_bytes": 73728, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 5242880, "logical_shape": [ 2048, 2560 ], "module": "text_embedder.linear_1", "original_bytes": 10485760, "parameter": "text_embedder.linear_1.weight", "re_quantize_for_matmul": true, "stored_bytes": 2949120, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "text_embedder.linear_2", "original_bytes": 8388608, "parameter": "text_embedder.linear_2.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.0.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.0.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.0.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.0.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.0.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.0.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.0.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.0.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.0.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.0.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.0.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.0.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.0.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.0.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.1.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.1.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.1.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.1.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.1.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.1.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.1.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.1.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.1.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.1.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.1.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.1.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.1.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.1.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.2.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.2.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.2.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.2.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.2.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.2.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.2.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.2.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.2.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.2.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.2.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.2.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.2.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.2.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.3.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.3.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.3.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.3.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.3.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.3.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.3.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.3.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.3.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.3.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.3.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.3.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.3.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.3.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.4.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.4.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.4.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.4.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.4.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.4.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.4.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.4.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.4.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.4.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.4.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.4.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.4.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.4.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.5.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.5.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.5.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.5.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.5.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.5.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.5.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.5.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.5.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.5.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.5.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.5.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.5.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.5.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.6.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.6.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.6.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.6.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.6.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.6.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.6.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.6.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.6.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.6.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.6.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.6.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.6.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.6.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.7.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.7.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.7.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.7.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.7.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.7.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.7.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.7.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.7.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.7.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.7.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.7.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.7.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.7.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.8.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.8.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.8.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.8.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.8.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.8.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.8.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.8.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.8.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.8.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.8.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.8.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.8.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.8.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.9.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.9.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.9.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.9.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.9.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.9.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.9.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.9.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.9.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.9.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.9.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.9.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.9.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.9.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.10.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.10.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.10.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.10.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.10.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.10.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.10.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.10.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.10.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.10.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.10.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.10.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.10.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.10.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.11.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.11.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.11.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.11.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.11.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.11.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.11.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.11.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.11.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.11.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.11.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.11.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.11.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.11.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.12.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.12.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.12.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.12.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.12.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.12.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.12.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.12.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.12.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.12.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.12.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.12.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.12.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.12.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.13.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.13.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.13.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.13.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.13.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.13.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.13.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.13.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.13.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.13.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.13.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.13.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.13.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.13.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.14.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.14.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.14.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.14.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.14.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.14.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.14.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.14.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.14.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.14.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.14.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.14.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.14.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.14.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.15.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.15.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.15.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.15.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.15.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.15.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.15.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.15.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.15.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.15.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.15.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.15.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.15.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.15.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.16.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.16.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.16.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.16.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.16.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.16.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.16.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.16.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.16.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.16.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.16.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.16.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.16.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.16.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.17.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.17.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.17.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.17.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.17.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.17.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.17.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.17.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.17.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.17.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.17.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.17.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.17.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.17.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.18.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.18.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.18.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.18.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.18.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.18.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.18.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.18.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.18.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.18.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.18.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.18.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.18.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.18.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.19.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.19.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.19.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.19.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.19.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.19.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.19.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.19.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.19.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.19.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.19.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.19.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.19.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.19.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.20.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.20.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.20.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.20.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.20.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.20.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.20.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.20.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.20.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.20.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.20.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.20.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.20.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.20.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.21.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.21.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.21.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.21.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.21.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.21.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.21.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.21.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.21.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.21.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.21.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.21.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.21.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.21.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.22.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.22.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.22.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.22.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.22.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.22.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.22.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.22.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.22.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.22.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.22.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.22.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.22.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.22.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.23.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.23.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.23.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.23.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.23.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.23.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.23.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.23.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.23.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.23.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.23.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.23.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.23.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.23.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.24.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.24.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.24.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.24.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.24.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.24.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.24.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.24.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.24.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.24.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.24.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.24.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.24.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.24.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.25.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.25.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.25.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.25.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.25.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.25.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.25.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.25.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.25.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.25.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.25.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.25.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.25.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.25.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.26.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.26.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.26.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.26.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.26.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.26.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.26.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.26.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.26.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.26.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.26.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.26.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.26.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.26.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.27.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.27.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.27.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.27.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.27.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.27.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.27.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.27.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.27.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.27.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.27.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.27.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.27.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.27.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.28.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.28.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.28.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.28.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.28.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.28.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.28.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.28.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.28.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.28.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.28.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.28.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.28.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.28.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.29.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.29.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.29.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.29.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.29.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.29.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.29.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.29.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.29.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.29.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.29.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.29.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.29.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.29.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.30.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.30.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.30.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.30.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.30.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.30.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.30.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.30.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.30.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.30.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.30.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.30.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.30.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.30.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.31.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.31.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.31.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.31.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.31.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.31.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.31.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.31.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.31.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.31.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.31.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.31.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.31.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.31.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.32.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.32.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.32.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.32.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.32.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.32.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.32.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.32.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.32.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.32.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.32.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.32.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.32.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.32.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.33.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.33.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.33.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.33.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.33.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.33.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.33.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.33.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.33.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.33.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.33.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.33.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.33.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.33.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.34.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.34.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.34.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.34.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.34.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.34.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.34.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.34.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.34.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.34.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.34.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.34.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.34.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.34.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.35.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.35.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.35.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.35.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.35.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.35.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.35.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.35.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.35.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.35.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.35.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.35.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.35.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.35.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.36.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.36.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.36.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.36.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.36.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.36.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.36.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.36.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.36.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.36.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.36.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.36.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.36.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.36.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.37.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.37.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.37.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.37.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.37.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.37.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.37.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.37.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.37.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.37.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.37.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.37.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.37.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.37.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.38.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.38.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.38.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.38.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.38.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.38.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.38.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.38.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.38.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.38.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.38.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.38.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.38.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.38.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.39.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.39.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.39.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.39.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.39.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.39.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.39.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.39.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.39.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.39.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.39.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.39.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.39.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.39.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.40.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.40.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.40.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.40.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.40.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.40.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.40.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.40.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.40.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.40.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.40.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.40.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.40.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.40.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.41.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.41.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.41.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.41.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.41.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.41.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.41.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.41.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.41.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.41.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.41.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.41.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.41.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.41.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.42.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.42.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.42.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.42.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.42.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.42.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.42.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.42.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.42.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.42.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.42.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.42.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.42.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.42.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.43.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.43.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.43.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.43.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.43.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.43.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.43.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.43.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.43.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.43.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.43.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.43.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.43.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.43.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.44.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.44.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.44.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.44.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.44.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.44.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.44.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.44.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.44.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.44.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.44.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.44.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.44.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.44.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.45.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.45.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.45.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.45.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.45.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.45.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.45.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.45.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.45.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.45.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.45.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.45.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.45.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.45.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.46.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.46.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.46.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.46.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.46.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.46.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.46.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.46.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.46.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.46.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.46.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.46.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.46.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.46.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.47.attn.to_q", "original_bytes": 8388608, "parameter": "blocks.47.attn.to_q.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.47.attn.to_k", "original_bytes": 8388608, "parameter": "blocks.47.attn.to_k.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.47.attn.to_v", "original_bytes": 8388608, "parameter": "blocks.47.attn.to_v.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 4194304, "logical_shape": [ 2048, 2048 ], "module": "blocks.47.attn.to_out", "original_bytes": 8388608, "parameter": "blocks.47.attn.to_out.weight", "re_quantize_for_matmul": true, "stored_bytes": 2359296, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.47.ffn.shared_experts.gate_proj", "original_bytes": 3145728, "parameter": "blocks.47.ffn.shared_experts.gate_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 768, 2048 ], "module": "blocks.47.ffn.shared_experts.up_proj", "original_bytes": 3145728, "parameter": "blocks.47.ffn.shared_experts.up_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" }, { "group_size": 64, "logical_numel": 1572864, "logical_shape": [ 2048, 768 ], "module": "blocks.47.ffn.shared_experts.down_proj", "original_bytes": 3145728, "parameter": "blocks.47.ffn.shared_experts.down_proj.weight", "re_quantize_for_matmul": true, "stored_bytes": 884736, "use_quantized_matmul": false, "weights_dtype": "uint4", "wrapper_class": "SDNQLinear" } ], "logical_params": 1041367040, "original_parameter_bytes": 2082734080, "stored_bytes": 585768960 }, "lingbot_video_code_revision": "a2bb04b78edd848500dc27a26e035a95442ae186", "moe_experts": { "compression_ratio": 3.2, "entries": [ { "module": "blocks.0.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.1.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.2.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.3.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.4.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.5.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.6.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.7.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.8.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.9.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.10.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.11.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.12.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.13.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.14.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.15.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.16.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.17.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.18.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.19.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.20.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.21.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.22.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.23.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.24.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.25.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.26.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.27.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.28.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.29.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.30.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.31.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.32.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.33.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.34.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.35.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.36.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.37.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.38.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.39.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.40.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.41.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.42.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.43.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.44.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.45.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.46.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } }, { "module": "blocks.47.ffn.experts", "weights": { "w1": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] }, "w2": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 2048, 768 ], "original_stride": [ 1572864, 768, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 2048, 24, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 2048, 768 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 2048, 24, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 2048, 24, 1 ] }, "w3": { "group_size": 32, "hadamard_group_size": 256, "layer_class_name": null, "logical_numel": 201326592, "original_bytes": 402653184, "original_numel": 201326592, "original_shape": [ 128, 768, 2048 ], "original_stride": [ 1572864, 2048, 1 ], "quantized_matmul_dtype": "int8", "quantized_weight_shape": [ 128, 768, 64, 32 ], "re_quantize_for_matmul": true, "result_dtype": "bfloat16", "result_shape": [ 128, 768, 2048 ], "scale_dtype": "bfloat16", "scale_shape": [ 128, 768, 64, 1 ], "stored_bytes": 125829120, "stored_weight_dtype": "uint8", "stored_weight_shape": [ 100663296 ], "svd_rank": 32, "svd_steps": 8, "use_hadamard": false, "use_quantized_matmul": false, "use_stochastic_rounding": false, "weights_dtype": "uint4", "zero_point_dtype": "bfloat16", "zero_point_shape": [ 128, 768, 64, 1 ] } } } ], "logical_numel": 28991029248, "original_bytes": 57982058496, "stored_bytes": 18119393280, "tensor_count": 144 }, "quantization_seconds": 27.26588556705974, "quantized": { "logical_params": 30032396288, "original_byte_coverage_percent": 99.65495460853698, "original_parameter_bytes_covered": 60064792576, "parameter_coverage_percent": 99.8267882886455, "stored_bytes": 18705162240 }, "recipe": { "expert_runtime": "SDNQ dequantize to BF16 plus SGLang Triton fused MoE", "generic_use_quantized_matmul": false, "group_size": 0, "name": "uint4-static-transformer-only-with-3d-expert-adapter", "quant_conv": false, "quant_embedding": false, "use_dynamic_quantization": false, "use_hadamard": false, "use_svd": false, "weights_dtype": "uint4" }, "saved_component_bytes": 18913776123, "sdnq_describe": "v0.2.1-17-gd841c38", "sdnq_revision": "d841c383ff7be38728d4df829e17af4f15d4fd66", "source_repo_id": "robbyant/lingbot-video-moe-30b-a3b", "source_revision": "f2e538f64afe00cc4ae674db2aeb52e2945edfd5", "torch": "2.8.0+cu128", "total": { "logical_params": 30084506176, "original_parameter_bytes": 60272760960 }, "unquantized": { "largest_parameters": [ { "bytes": 100663296, "dtype": "float32", "numel": 25165824, "parameter": "time_modulation.1.weight", "shape": [ 12288, 2048 ] }, { "bytes": 33554432, "dtype": "float32", "numel": 8388608, "parameter": "norm_out_modulation.1.weight", "shape": [ 4096, 2048 ] }, { "bytes": 16777216, "dtype": "float32", "numel": 4194304, "parameter": "time_embedder.linear_2.weight", "shape": [ 2048, 2048 ] }, { "bytes": 2097152, "dtype": "float32", "numel": 524288, "parameter": "time_embedder.linear_1.weight", "shape": [ 2048, 256 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.0.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.1.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.2.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.3.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.4.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.5.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.6.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.7.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.8.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.9.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.10.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.11.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.12.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.13.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.14.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.15.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.16.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.17.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.18.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.19.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.20.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.21.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.22.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.23.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.24.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.25.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.26.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.27.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.28.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.29.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.30.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.31.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.32.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.33.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.34.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.35.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.36.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.37.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.38.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.39.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.40.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.41.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.42.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.43.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.44.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.45.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.46.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 1048576, "dtype": "float32", "numel": 262144, "parameter": "blocks.47.ffn.router.weight", "shape": [ 128, 2048 ] }, { "bytes": 262144, "dtype": "bfloat16", "numel": 131072, "parameter": "proj_out.weight", "shape": [ 64, 2048 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "time_modulation.1.bias", "shape": [ 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.0.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.1.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.2.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.3.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.4.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.5.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.6.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.7.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.8.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.9.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.10.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.11.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.12.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.13.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.14.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.15.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.16.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.17.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.18.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.19.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.20.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.21.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.22.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.23.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.24.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.25.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.26.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.27.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.28.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.29.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.30.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.31.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.32.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.33.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.34.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.35.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.36.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.37.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.38.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.39.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.40.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.41.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.42.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.43.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.44.scale_shift_table", "shape": [ 1, 12288 ] }, { "bytes": 49152, "dtype": "float32", "numel": 12288, "parameter": "blocks.45.scale_shift_table", "shape": [ 1, 12288 ] } ], "logical_params": 52109888, "original_parameter_bytes": 207968384, "parameter_count": 446 } }