{
  "method": "awq",
  "recipe": {
    "quant_cfg": [
      {
        "quantizer_name": "*",
        "enable": false
      },
      {
        "quantizer_name": "*weight_quantizer",
        "cfg": {
          "num_bits": 4,
          "block_sizes": {
            "-1": 128,
            "type": "static"
          }
        }
      },
      {
        "quantizer_name": "*input_quantizer",
        "enable": false
      },
      {
        "quantizer_name": "*block_sparse_moe.gate*",
        "enable": false
      },
      {
        "quantizer_name": "*linear_attn.conv1d*",
        "enable": false
      },
      {
        "quantizer_name": "*self_attention.conv1d*",
        "enable": false
      },
      {
        "quantizer_name": "*linear_attn.in_proj_a*",
        "enable": false
      },
      {
        "quantizer_name": "*linear_attn.in_proj_b*",
        "enable": false
      },
      {
        "quantizer_name": "*lm_head*",
        "enable": false
      },
      {
        "quantizer_name": "*mixer.conv1d*",
        "enable": false
      },
      {
        "quantizer_name": "*mlp.gate.*",
        "enable": false
      },
      {
        "quantizer_name": "*mlp.shared_expert_gate.*",
        "enable": false
      },
      {
        "quantizer_name": "*output_layer*",
        "enable": false
      },
      {
        "quantizer_name": "*proj_out.*",
        "enable": false
      },
      {
        "quantizer_name": "*router*",
        "enable": false
      },
      {
        "quantizer_name": "mtp.*",
        "enable": false
      },
      {
        "quantizer_name": "output.*",
        "enable": false
      },
      {
        "quantizer_name": "*embed_vision*",
        "enable": false
      },
      {
        "quantizer_name": "*vision_tower*",
        "enable": false
      },
      {
        "quantizer_name": "*visual*",
        "enable": false
      },
      {
        "quantizer_name": "*vision_model*",
        "enable": false
      },
      {
        "quantizer_name": "*multi_modal_projector*",
        "enable": false
      },
      {
        "quantizer_name": "*",
        "parent_class": "nn.BatchNorm1d",
        "enable": false
      },
      {
        "quantizer_name": "*",
        "parent_class": "nn.BatchNorm2d",
        "enable": false
      },
      {
        "quantizer_name": "*",
        "parent_class": "nn.BatchNorm3d",
        "enable": false
      },
      {
        "quantizer_name": "*",
        "parent_class": "nn.LeakyReLU",
        "enable": false
      },
      {
        "quantizer_name": "*",
        "parent_class": "nn.Embedding",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.0.linear_attn.in_proj_b.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.0.linear_attn.in_proj_a.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.1.linear_attn.in_proj_b.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.1.linear_attn.in_proj_a.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.2.linear_attn.in_proj_b.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.2.linear_attn.in_proj_a.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.3.memory.admission_down.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.3.memory.admission_up.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.4.linear_attn.in_proj_b.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.4.linear_attn.in_proj_a.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.5.linear_attn.in_proj_b.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.5.linear_attn.in_proj_a.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.6.linear_attn.in_proj_b.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.6.linear_attn.in_proj_a.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.7.memory.admission_down.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.7.memory.admission_up.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.8.linear_attn.in_proj_b.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.8.linear_attn.in_proj_a.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.9.linear_attn.in_proj_b.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.9.linear_attn.in_proj_a.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.10.linear_attn.in_proj_b.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.10.linear_attn.in_proj_a.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.11.memory.admission_down.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.11.memory.admission_up.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.12.linear_attn.in_proj_b.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.12.linear_attn.in_proj_a.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.13.linear_attn.in_proj_b.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.13.linear_attn.in_proj_a.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.14.linear_attn.in_proj_b.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.14.linear_attn.in_proj_a.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.15.memory.admission_down.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.15.memory.admission_up.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.16.linear_attn.in_proj_b.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.16.linear_attn.in_proj_a.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.17.linear_attn.in_proj_b.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.17.linear_attn.in_proj_a.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.18.linear_attn.in_proj_b.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.18.linear_attn.in_proj_a.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.19.memory.admission_down.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.19.memory.admission_up.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.20.linear_attn.in_proj_b.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.20.linear_attn.in_proj_a.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.21.linear_attn.in_proj_b.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.21.linear_attn.in_proj_a.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.22.linear_attn.in_proj_b.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.22.linear_attn.in_proj_a.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.23.memory.admission_down.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.23.memory.admission_up.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.24.linear_attn.in_proj_b.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.24.linear_attn.in_proj_a.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.25.linear_attn.in_proj_b.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.25.linear_attn.in_proj_a.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.26.linear_attn.in_proj_b.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.26.linear_attn.in_proj_a.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.27.memory.admission_down.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.27.memory.admission_up.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.28.linear_attn.in_proj_b.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.28.linear_attn.in_proj_a.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.29.linear_attn.in_proj_b.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.29.linear_attn.in_proj_a.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.30.linear_attn.in_proj_b.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.30.linear_attn.in_proj_a.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.31.memory.admission_down.*",
        "enable": false
      },
      {
        "quantizer_name": "model.layers.31.memory.admission_up.*",
        "enable": false
      },
      {
        "quantizer_name": "lm_head.*",
        "enable": false
      }
    ],
    "algorithm": {
      "method": "awq_lite",
      "alpha_step": 0.1
    }
  },
  "quantized_modules": 256,
  "maximum_weight_repacking_error": 0.0,
  "maximum_kernel_relative_error": 0.0049504950495049506,
  "weight_bytes": 3398133304,
  "seconds": 353.6774011319503,
  "versions": {
    "torch": "2.11.0",
    "transformers": "5.7.0.dev0",
    "nvidia-modelopt": "0.47.0",
    "triton": "3.6.0",
    "accelerate": "1.13.0",
    "safetensors": "0.7.0",
    "huggingface-hub": "1.5.0"
  },
  "hardware": "NVIDIA H800"
}