quantization_config.json

{
  "recipe": "fp8_blockwise_mixed",
  "weight_only": true,
  "quant_lmhead": false,
  "mixed_precision": {
    "quantized": [
      "mlp.*",
      "mlp_moe_gen.*"
    ],
    "bf16_kept": [
      "self_attn.*"
    ],
    "n_quantized": 216
  },
  "scale_layout": {
    "granularity": "blockwise-128x128",
    "block_sizes": {
      "rows": 128,
      "cols": 128
    },
    "n_total_weight": 802,
    "n_quantized_weight": 216,
    "n_scale": 432,
    "total_scale_elements": 625568,
    "scale_overhead_pct": 0.0046,
    "example_shapes": {
      "12288x4096": {
        "weight_shape": [
          12288,
          4096
        ],
        "amax_shape": [
          96,
          1,
          32,
          1
        ],
        "module_example": "layers.0.mlp.gate_proj"
      },
      "4096x12288": {
        "weight_shape": [
          4096,
          12288
        ],
        "amax_shape": [
          32,
          1,
          96,
          1
        ],
        "module_example": "layers.0.mlp.down_proj"
      }
    }
  },
  "loader": "from_config + restore_from_modelopt_state + safetensors load_state_dict",
  "sidecar": "transformer/modelopt_state.pt"
}