| { |
| "method": "awq", |
| "recipe": { |
| "quant_cfg": [ |
| { |
| "quantizer_name": "*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "*weight_quantizer", |
| "cfg": { |
| "num_bits": 4, |
| "block_sizes": { |
| "-1": 128, |
| "type": "static" |
| } |
| } |
| }, |
| { |
| "quantizer_name": "*input_quantizer", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "*block_sparse_moe.gate*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "*linear_attn.conv1d*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "*self_attention.conv1d*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "*linear_attn.in_proj_a*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "*linear_attn.in_proj_b*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "*lm_head*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "*mixer.conv1d*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "*mlp.gate.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "*mlp.shared_expert_gate.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "*output_layer*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "*proj_out.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "*router*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "mtp.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "output.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "*embed_vision*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "*vision_tower*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "*visual*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "*vision_model*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "*multi_modal_projector*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "*", |
| "parent_class": "nn.BatchNorm1d", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "*", |
| "parent_class": "nn.BatchNorm2d", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "*", |
| "parent_class": "nn.BatchNorm3d", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "*", |
| "parent_class": "nn.LeakyReLU", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "*", |
| "parent_class": "nn.Embedding", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.0.linear_attn.in_proj_b.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.0.linear_attn.in_proj_a.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.1.linear_attn.in_proj_b.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.1.linear_attn.in_proj_a.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.2.linear_attn.in_proj_b.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.2.linear_attn.in_proj_a.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.3.memory.admission_down.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.3.memory.admission_up.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.4.linear_attn.in_proj_b.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.4.linear_attn.in_proj_a.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.5.linear_attn.in_proj_b.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.5.linear_attn.in_proj_a.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.6.linear_attn.in_proj_b.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.6.linear_attn.in_proj_a.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.7.memory.admission_down.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.7.memory.admission_up.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.8.linear_attn.in_proj_b.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.8.linear_attn.in_proj_a.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.9.linear_attn.in_proj_b.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.9.linear_attn.in_proj_a.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.10.linear_attn.in_proj_b.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.10.linear_attn.in_proj_a.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.11.memory.admission_down.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.11.memory.admission_up.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.12.linear_attn.in_proj_b.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.12.linear_attn.in_proj_a.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.13.linear_attn.in_proj_b.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.13.linear_attn.in_proj_a.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.14.linear_attn.in_proj_b.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.14.linear_attn.in_proj_a.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.15.memory.admission_down.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.15.memory.admission_up.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.16.linear_attn.in_proj_b.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.16.linear_attn.in_proj_a.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.17.linear_attn.in_proj_b.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.17.linear_attn.in_proj_a.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.18.linear_attn.in_proj_b.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.18.linear_attn.in_proj_a.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.19.memory.admission_down.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.19.memory.admission_up.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.20.linear_attn.in_proj_b.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.20.linear_attn.in_proj_a.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.21.linear_attn.in_proj_b.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.21.linear_attn.in_proj_a.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.22.linear_attn.in_proj_b.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.22.linear_attn.in_proj_a.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.23.memory.admission_down.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.23.memory.admission_up.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.24.linear_attn.in_proj_b.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.24.linear_attn.in_proj_a.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.25.linear_attn.in_proj_b.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.25.linear_attn.in_proj_a.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.26.linear_attn.in_proj_b.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.26.linear_attn.in_proj_a.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.27.memory.admission_down.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.27.memory.admission_up.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.28.linear_attn.in_proj_b.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.28.linear_attn.in_proj_a.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.29.linear_attn.in_proj_b.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.29.linear_attn.in_proj_a.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.30.linear_attn.in_proj_b.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.30.linear_attn.in_proj_a.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.31.memory.admission_down.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "model.layers.31.memory.admission_up.*", |
| "enable": false |
| }, |
| { |
| "quantizer_name": "lm_head.*", |
| "enable": false |
| } |
| ], |
| "algorithm": { |
| "method": "awq_lite", |
| "alpha_step": 0.1 |
| } |
| }, |
| "quantized_modules": 256, |
| "maximum_weight_repacking_error": 0.0, |
| "maximum_kernel_relative_error": 0.0049504950495049506, |
| "weight_bytes": 3398133304, |
| "seconds": 353.6774011319503, |
| "versions": { |
| "torch": "2.11.0", |
| "transformers": "5.7.0.dev0", |
| "nvidia-modelopt": "0.47.0", |
| "triton": "3.6.0", |
| "accelerate": "1.13.0", |
| "safetensors": "0.7.0", |
| "huggingface-hub": "1.5.0" |
| }, |
| "hardware": "NVIDIA H800" |
| } |
|
|
