| { |
| "component": "refiner", |
| "cuda_peak_allocated_bytes": 2239758336, |
| "cuda_peak_reserved_bytes": 2277507072, |
| "generic_modules": { |
| "actual_group_size_counts": { |
| "-1": 1, |
| "64": 338 |
| }, |
| "count": 339, |
| "entries": [ |
| { |
| "group_size": -1, |
| "logical_numel": 131072, |
| "logical_shape": [ |
| 2048, |
| 64 |
| ], |
| "module": "patch_embedder", |
| "original_bytes": 262144, |
| "parameter": "patch_embedder.weight", |
| "re_quantize_for_matmul": false, |
| "stored_bytes": 73728, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 5242880, |
| "logical_shape": [ |
| 2048, |
| 2560 |
| ], |
| "module": "text_embedder.linear_1", |
| "original_bytes": 10485760, |
| "parameter": "text_embedder.linear_1.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2949120, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "text_embedder.linear_2", |
| "original_bytes": 8388608, |
| "parameter": "text_embedder.linear_2.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.0.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.0.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.0.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.0.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.0.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.0.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.0.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.0.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.0.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.0.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.0.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.0.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.0.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.0.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.1.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.1.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.1.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.1.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.1.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.1.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.1.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.1.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.1.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.1.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.1.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.1.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.1.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.1.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.2.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.2.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.2.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.2.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.2.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.2.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.2.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.2.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.2.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.2.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.2.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.2.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.2.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.2.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.3.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.3.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.3.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.3.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.3.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.3.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.3.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.3.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.3.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.3.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.3.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.3.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.3.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.3.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.4.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.4.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.4.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.4.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.4.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.4.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.4.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.4.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.4.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.4.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.4.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.4.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.4.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.4.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.5.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.5.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.5.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.5.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.5.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.5.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.5.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.5.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.5.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.5.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.5.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.5.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.5.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.5.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.6.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.6.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.6.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.6.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.6.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.6.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.6.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.6.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.6.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.6.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.6.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.6.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.6.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.6.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.7.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.7.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.7.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.7.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.7.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.7.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.7.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.7.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.7.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.7.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.7.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.7.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.7.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.7.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.8.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.8.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.8.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.8.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.8.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.8.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.8.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.8.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.8.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.8.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.8.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.8.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.8.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.8.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.9.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.9.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.9.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.9.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.9.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.9.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.9.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.9.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.9.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.9.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.9.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.9.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.9.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.9.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.10.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.10.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.10.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.10.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.10.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.10.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.10.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.10.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.10.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.10.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.10.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.10.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.10.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.10.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.11.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.11.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.11.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.11.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.11.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.11.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.11.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.11.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.11.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.11.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.11.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.11.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.11.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.11.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.12.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.12.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.12.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.12.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.12.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.12.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.12.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.12.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.12.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.12.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.12.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.12.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.12.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.12.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.13.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.13.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.13.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.13.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.13.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.13.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.13.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.13.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.13.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.13.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.13.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.13.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.13.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.13.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.14.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.14.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.14.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.14.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.14.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.14.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.14.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.14.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.14.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.14.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.14.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.14.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.14.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.14.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.15.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.15.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.15.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.15.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.15.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.15.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.15.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.15.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.15.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.15.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.15.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.15.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.15.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.15.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.16.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.16.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.16.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.16.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.16.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.16.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.16.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.16.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.16.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.16.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.16.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.16.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.16.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.16.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.17.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.17.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.17.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.17.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.17.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.17.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.17.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.17.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.17.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.17.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.17.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.17.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.17.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.17.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.18.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.18.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.18.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.18.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.18.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.18.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.18.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.18.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.18.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.18.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.18.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.18.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.18.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.18.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.19.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.19.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.19.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.19.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.19.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.19.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.19.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.19.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.19.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.19.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.19.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.19.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.19.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.19.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.20.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.20.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.20.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.20.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.20.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.20.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.20.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.20.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.20.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.20.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.20.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.20.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.20.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.20.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.21.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.21.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.21.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.21.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.21.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.21.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.21.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.21.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.21.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.21.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.21.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.21.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.21.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.21.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.22.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.22.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.22.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.22.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.22.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.22.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.22.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.22.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.22.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.22.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.22.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.22.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.22.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.22.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.23.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.23.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.23.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.23.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.23.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.23.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.23.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.23.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.23.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.23.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.23.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.23.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.23.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.23.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.24.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.24.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.24.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.24.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.24.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.24.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.24.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.24.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.24.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.24.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.24.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.24.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.24.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.24.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.25.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.25.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.25.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.25.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.25.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.25.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.25.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.25.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.25.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.25.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.25.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.25.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.25.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.25.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.26.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.26.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.26.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.26.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.26.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.26.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.26.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.26.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.26.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.26.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.26.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.26.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.26.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.26.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.27.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.27.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.27.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.27.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.27.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.27.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.27.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.27.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.27.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.27.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.27.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.27.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.27.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.27.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.28.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.28.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.28.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.28.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.28.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.28.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.28.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.28.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.28.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.28.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.28.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.28.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.28.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.28.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.29.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.29.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.29.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.29.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.29.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.29.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.29.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.29.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.29.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.29.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.29.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.29.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.29.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.29.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.30.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.30.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.30.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.30.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.30.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.30.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.30.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.30.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.30.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.30.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.30.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.30.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.30.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.30.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.31.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.31.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.31.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.31.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.31.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.31.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.31.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.31.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.31.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.31.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.31.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.31.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.31.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.31.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.32.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.32.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.32.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.32.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.32.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.32.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.32.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.32.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.32.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.32.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.32.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.32.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.32.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.32.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.33.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.33.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.33.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.33.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.33.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.33.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.33.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.33.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.33.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.33.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.33.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.33.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.33.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.33.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.34.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.34.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.34.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.34.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.34.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.34.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.34.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.34.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.34.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.34.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.34.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.34.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.34.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.34.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.35.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.35.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.35.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.35.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.35.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.35.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.35.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.35.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.35.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.35.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.35.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.35.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.35.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.35.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.36.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.36.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.36.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.36.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.36.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.36.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.36.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.36.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.36.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.36.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.36.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.36.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.36.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.36.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.37.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.37.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.37.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.37.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.37.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.37.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.37.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.37.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.37.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.37.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.37.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.37.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.37.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.37.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.38.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.38.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.38.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.38.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.38.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.38.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.38.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.38.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.38.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.38.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.38.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.38.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.38.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.38.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.39.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.39.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.39.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.39.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.39.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.39.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.39.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.39.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.39.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.39.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.39.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.39.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.39.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.39.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.40.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.40.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.40.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.40.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.40.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.40.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.40.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.40.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.40.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.40.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.40.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.40.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.40.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.40.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.41.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.41.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.41.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.41.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.41.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.41.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.41.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.41.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.41.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.41.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.41.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.41.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.41.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.41.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.42.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.42.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.42.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.42.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.42.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.42.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.42.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.42.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.42.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.42.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.42.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.42.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.42.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.42.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.43.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.43.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.43.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.43.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.43.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.43.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.43.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.43.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.43.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.43.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.43.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.43.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.43.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.43.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.44.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.44.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.44.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.44.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.44.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.44.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.44.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.44.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.44.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.44.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.44.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.44.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.44.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.44.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.45.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.45.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.45.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.45.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.45.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.45.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.45.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.45.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.45.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.45.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.45.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.45.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.45.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.45.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.46.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.46.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.46.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.46.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.46.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.46.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.46.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.46.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.46.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.46.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.46.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.46.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.46.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.46.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.47.attn.to_q", |
| "original_bytes": 8388608, |
| "parameter": "blocks.47.attn.to_q.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.47.attn.to_k", |
| "original_bytes": 8388608, |
| "parameter": "blocks.47.attn.to_k.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.47.attn.to_v", |
| "original_bytes": 8388608, |
| "parameter": "blocks.47.attn.to_v.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 4194304, |
| "logical_shape": [ |
| 2048, |
| 2048 |
| ], |
| "module": "blocks.47.attn.to_out", |
| "original_bytes": 8388608, |
| "parameter": "blocks.47.attn.to_out.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 2359296, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.47.ffn.shared_experts.gate_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.47.ffn.shared_experts.gate_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 768, |
| 2048 |
| ], |
| "module": "blocks.47.ffn.shared_experts.up_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.47.ffn.shared_experts.up_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| }, |
| { |
| "group_size": 64, |
| "logical_numel": 1572864, |
| "logical_shape": [ |
| 2048, |
| 768 |
| ], |
| "module": "blocks.47.ffn.shared_experts.down_proj", |
| "original_bytes": 3145728, |
| "parameter": "blocks.47.ffn.shared_experts.down_proj.weight", |
| "re_quantize_for_matmul": true, |
| "stored_bytes": 884736, |
| "use_quantized_matmul": false, |
| "weights_dtype": "uint4", |
| "wrapper_class": "SDNQLinear" |
| } |
| ], |
| "logical_params": 1041367040, |
| "original_parameter_bytes": 2082734080, |
| "stored_bytes": 585768960 |
| }, |
| "lingbot_video_code_revision": "a2bb04b78edd848500dc27a26e035a95442ae186", |
| "moe_experts": { |
| "compression_ratio": 3.2, |
| "entries": [ |
| { |
| "module": "blocks.0.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.1.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.2.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.3.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.4.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.5.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.6.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.7.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.8.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.9.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.10.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.11.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.12.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.13.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.14.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.15.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.16.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.17.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.18.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.19.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.20.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.21.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.22.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.23.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.24.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.25.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.26.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.27.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.28.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.29.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.30.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.31.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.32.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.33.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.34.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.35.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.36.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.37.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.38.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.39.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.40.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.41.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.42.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.43.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.44.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.45.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.46.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| }, |
| { |
| "module": "blocks.47.ffn.experts", |
| "weights": { |
| "w1": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| }, |
| "w2": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "original_stride": [ |
| 1572864, |
| 768, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 2048, |
| 24, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 2048, |
| 768 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 2048, |
| 24, |
| 1 |
| ] |
| }, |
| "w3": { |
| "group_size": 32, |
| "hadamard_group_size": 256, |
| "layer_class_name": null, |
| "logical_numel": 201326592, |
| "original_bytes": 402653184, |
| "original_numel": 201326592, |
| "original_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "original_stride": [ |
| 1572864, |
| 2048, |
| 1 |
| ], |
| "quantized_matmul_dtype": "int8", |
| "quantized_weight_shape": [ |
| 128, |
| 768, |
| 64, |
| 32 |
| ], |
| "re_quantize_for_matmul": true, |
| "result_dtype": "bfloat16", |
| "result_shape": [ |
| 128, |
| 768, |
| 2048 |
| ], |
| "scale_dtype": "bfloat16", |
| "scale_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ], |
| "stored_bytes": 125829120, |
| "stored_weight_dtype": "uint8", |
| "stored_weight_shape": [ |
| 100663296 |
| ], |
| "svd_rank": 32, |
| "svd_steps": 8, |
| "use_hadamard": false, |
| "use_quantized_matmul": false, |
| "use_stochastic_rounding": false, |
| "weights_dtype": "uint4", |
| "zero_point_dtype": "bfloat16", |
| "zero_point_shape": [ |
| 128, |
| 768, |
| 64, |
| 1 |
| ] |
| } |
| } |
| } |
| ], |
| "logical_numel": 28991029248, |
| "original_bytes": 57982058496, |
| "stored_bytes": 18119393280, |
| "tensor_count": 144 |
| }, |
| "quantization_seconds": 27.26588556705974, |
| "quantized": { |
| "logical_params": 30032396288, |
| "original_byte_coverage_percent": 99.65495460853698, |
| "original_parameter_bytes_covered": 60064792576, |
| "parameter_coverage_percent": 99.8267882886455, |
| "stored_bytes": 18705162240 |
| }, |
| "recipe": { |
| "expert_runtime": "SDNQ dequantize to BF16 plus SGLang Triton fused MoE", |
| "generic_use_quantized_matmul": false, |
| "group_size": 0, |
| "name": "uint4-static-transformer-only-with-3d-expert-adapter", |
| "quant_conv": false, |
| "quant_embedding": false, |
| "use_dynamic_quantization": false, |
| "use_hadamard": false, |
| "use_svd": false, |
| "weights_dtype": "uint4" |
| }, |
| "saved_component_bytes": 18913776123, |
| "sdnq_describe": "v0.2.1-17-gd841c38", |
| "sdnq_revision": "d841c383ff7be38728d4df829e17af4f15d4fd66", |
| "source_repo_id": "robbyant/lingbot-video-moe-30b-a3b", |
| "source_revision": "f2e538f64afe00cc4ae674db2aeb52e2945edfd5", |
| "torch": "2.8.0+cu128", |
| "total": { |
| "logical_params": 30084506176, |
| "original_parameter_bytes": 60272760960 |
| }, |
| "unquantized": { |
| "largest_parameters": [ |
| { |
| "bytes": 100663296, |
| "dtype": "float32", |
| "numel": 25165824, |
| "parameter": "time_modulation.1.weight", |
| "shape": [ |
| 12288, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 33554432, |
| "dtype": "float32", |
| "numel": 8388608, |
| "parameter": "norm_out_modulation.1.weight", |
| "shape": [ |
| 4096, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 16777216, |
| "dtype": "float32", |
| "numel": 4194304, |
| "parameter": "time_embedder.linear_2.weight", |
| "shape": [ |
| 2048, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 2097152, |
| "dtype": "float32", |
| "numel": 524288, |
| "parameter": "time_embedder.linear_1.weight", |
| "shape": [ |
| 2048, |
| 256 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.0.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.1.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.2.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.3.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.4.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.5.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.6.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.7.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.8.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.9.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.10.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.11.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.12.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.13.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.14.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.15.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.16.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.17.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.18.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.19.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.20.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.21.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.22.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.23.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.24.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.25.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.26.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.27.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.28.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.29.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.30.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.31.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.32.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.33.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.34.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.35.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.36.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.37.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.38.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.39.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.40.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.41.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.42.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.43.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.44.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.45.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.46.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 1048576, |
| "dtype": "float32", |
| "numel": 262144, |
| "parameter": "blocks.47.ffn.router.weight", |
| "shape": [ |
| 128, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 262144, |
| "dtype": "bfloat16", |
| "numel": 131072, |
| "parameter": "proj_out.weight", |
| "shape": [ |
| 64, |
| 2048 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "time_modulation.1.bias", |
| "shape": [ |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.0.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.1.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.2.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.3.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.4.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.5.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.6.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.7.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.8.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.9.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.10.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.11.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.12.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.13.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.14.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.15.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.16.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.17.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.18.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.19.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.20.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.21.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.22.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.23.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.24.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.25.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.26.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.27.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.28.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.29.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.30.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.31.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.32.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.33.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.34.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.35.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.36.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.37.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.38.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.39.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.40.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.41.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.42.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.43.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.44.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| }, |
| { |
| "bytes": 49152, |
| "dtype": "float32", |
| "numel": 12288, |
| "parameter": "blocks.45.scale_shift_table", |
| "shape": [ |
| 1, |
| 12288 |
| ] |
| } |
| ], |
| "logical_params": 52109888, |
| "original_parameter_bytes": 207968384, |
| "parameter_count": 446 |
| } |
| } |
|
|
