diff --git "a/config.json" "b/config.json" --- "a/config.json" +++ "b/config.json" @@ -6,6 +6,7 @@ "attention_dropout": 0.0, "bos_token_id": 0, "eos_token_id": 1, + "expert_dtype": "fp4", "hc_eps": 1e-06, "hc_mult": 4, "hc_sinkhorn_iters": 20, @@ -480,43 +481,40 @@ "quant_method": "quark", "layer_type_quant_config": {}, "layer_quant_config": { - "mtp.0.ffn.experts.312.w3": { + "mtp.0.ffn.experts.62.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -524,7 +522,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.268.w1": { + "layers.7.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -568,43 +566,40 @@ "bias": null, "target_device": null }, - "layers.39.attn.wq_a": { + "mtp.0.ffn.experts.258.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -612,43 +607,40 @@ "bias": null, "target_device": null }, - "layers.53.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.266.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -656,7 +648,7 @@ "bias": null, "target_device": null }, - "layers.54.attn.wkv": { + "layers.56.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -700,43 +692,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.254.w2": { + "mtp.0.ffn.experts.286.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -744,7 +733,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.22.w1": { + "layers.14.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -788,7 +777,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.382.w3": { + "layers.0.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -832,43 +821,40 @@ "bias": null, "target_device": null }, - "layers.55.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.375.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -876,43 +862,40 @@ "bias": null, "target_device": null }, - "layers.12.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.155.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -920,43 +903,40 @@ "bias": null, "target_device": null }, - "layers.13.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.112.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -964,43 +944,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.344.w1": { + "mtp.0.ffn.experts.350.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -1008,43 +985,40 @@ "bias": null, "target_device": null }, - "layers.11.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.166.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -1052,43 +1026,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.39.w2": { + "mtp.0.ffn.experts.375.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -1096,43 +1067,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.29.w2": { + "mtp.0.ffn.experts.153.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -1140,43 +1108,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.366.w3": { + "mtp.0.ffn.experts.247.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -1184,7 +1149,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.161.w3": { + "layers.30.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -1228,43 +1193,40 @@ "bias": null, "target_device": null }, - "layers.5.attn.wkv": { + "mtp.0.ffn.experts.125.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -1272,7 +1234,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.122.w1": { + "layers.7.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -1316,43 +1278,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.352.w1": { + "mtp.0.ffn.experts.165.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -1360,43 +1319,40 @@ "bias": null, "target_device": null }, - "layers.31.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.6.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -1404,7 +1360,7 @@ "bias": null, "target_device": null }, - "layers.9.attn.wo_b": { + "layers.47.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -1448,43 +1404,40 @@ "bias": null, "target_device": null }, - "layers.13.attn.wo_a": { + "mtp.0.ffn.experts.209.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -1492,7 +1445,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.150.w3": { + "layers.22.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -1536,87 +1489,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.1.w3": { + "mtp.0.ffn.experts.316.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, - "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, - "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", - "is_scale_quant": false, - "enable_buffer_reuse": false, - "max_input_numel": 4194304 - }, - "bias": null, - "target_device": null - }, - "layers.10.ffn.shared_experts.w1": { - "input_tensors": { - "dtype": "fp8_e4m3", - "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, - "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, - "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", - "is_scale_quant": false, - "enable_buffer_reuse": false, - "max_input_numel": 4194304 - }, - "output_tensors": null, - "weight": { - "dtype": "fp8_e4m3", - "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -1624,7 +1530,7 @@ "bias": null, "target_device": null }, - "layers.56.attn.wq_b": { + "layers.41.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -1668,43 +1574,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.316.w3": { + "mtp.0.ffn.experts.288.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -1712,7 +1615,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.106.w1": { + "layers.52.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -1756,43 +1659,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.231.w1": { + "mtp.0.ffn.experts.209.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -1800,43 +1700,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.11.w1": { + "mtp.0.ffn.experts.106.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -1844,43 +1741,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.311.w3": { + "mtp.0.ffn.experts.159.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -1888,7 +1782,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.263.w3": { + "layers.5.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -1932,43 +1826,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.220.w2": { + "mtp.0.ffn.experts.358.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -1976,7 +1867,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.93.w1": { + "layers.39.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -2020,43 +1911,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.349.w1": { + "mtp.0.ffn.experts.193.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -2064,7 +1952,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.119.w1": { + "layers.30.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -2108,7 +1996,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.366.w1": { + "layers.44.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -2152,7 +2040,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.216.w2": { + "layers.26.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -2196,43 +2084,40 @@ "bias": null, "target_device": null }, - "layers.60.attn.wkv": { + "mtp.0.ffn.experts.355.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -2240,43 +2125,40 @@ "bias": null, "target_device": null }, - "layers.4.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.146.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -2284,43 +2166,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.362.w1": { + "mtp.0.ffn.experts.0.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -2328,43 +2207,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.83.w2": { + "mtp.0.ffn.experts.132.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -2372,43 +2248,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.100.w1": { + "mtp.0.ffn.experts.80.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -2416,43 +2289,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.66.w3": { + "mtp.0.ffn.experts.35.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -2460,7 +2330,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.118.w2": { + "layers.54.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -2504,7 +2374,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.137.w2": { + "layers.7.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -2548,43 +2418,40 @@ "bias": null, "target_device": null }, - "layers.18.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.266.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -2592,7 +2459,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.129.w1": { + "layers.22.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -2636,7 +2503,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.383.w3": { + "layers.37.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -2680,43 +2547,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.245.w1": { + "mtp.0.ffn.experts.234.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -2724,7 +2588,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.133.w2": { + "layers.37.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -2768,43 +2632,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.171.w1": { + "mtp.0.ffn.experts.114.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -2812,7 +2673,7 @@ "bias": null, "target_device": null }, - "layers.30.attn.wo_b": { + "layers.3.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -2856,7 +2717,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.187.w1": { + "layers.25.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -2900,7 +2761,7 @@ "bias": null, "target_device": null }, - "layers.31.attn.wq_b": { + "layers.16.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -2944,43 +2805,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.320.w2": { + "mtp.0.ffn.experts.319.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -2988,7 +2846,7 @@ "bias": null, "target_device": null }, - "layers.34.attn.wo_a": { + "layers.36.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -3032,7 +2890,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.81.w2": { + "layers.49.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -3076,7 +2934,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.332.w2": { + "layers.16.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -3120,43 +2978,40 @@ "bias": null, "target_device": null }, - "layers.44.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.154.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -3164,43 +3019,40 @@ "bias": null, "target_device": null }, - "layers.22.attn.wo_b": { + "mtp.0.ffn.experts.80.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -3208,7 +3060,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.282.w2": { + "layers.36.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -3252,43 +3104,40 @@ "bias": null, "target_device": null }, - "layers.32.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.307.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -3296,43 +3145,40 @@ "bias": null, "target_device": null }, - "layers.28.attn.wo_b": { + "mtp.0.ffn.experts.66.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -3340,43 +3186,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.336.w3": { + "mtp.0.ffn.experts.346.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -3384,7 +3227,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.267.w3": { + "layers.25.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -3428,7 +3271,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.170.w3": { + "layers.46.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -3472,7 +3315,7 @@ "bias": null, "target_device": null }, - "layers.28.attn.wq_a": { + "layers.37.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -3516,43 +3359,40 @@ "bias": null, "target_device": null }, - "layers.42.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.268.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -3560,43 +3400,40 @@ "bias": null, "target_device": null }, - "layers.20.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.133.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -3604,43 +3441,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.209.w1": { + "mtp.0.ffn.experts.338.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -3648,7 +3482,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.356.w1": { + "layers.29.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -3692,43 +3526,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.231.w3": { + "mtp.0.ffn.experts.4.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -3736,7 +3567,7 @@ "bias": null, "target_device": null }, - "layers.46.attn.wo_a": { + "mtp.0.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -3780,7 +3611,7 @@ "bias": null, "target_device": null }, - "layers.24.attn.wq_a": { + "layers.31.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -3824,43 +3655,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.261.w2": { + "mtp.0.ffn.experts.84.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -3868,43 +3696,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.266.w1": { + "mtp.0.ffn.experts.38.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -3912,43 +3737,40 @@ "bias": null, "target_device": null }, - "layers.5.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.365.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -3956,7 +3778,7 @@ "bias": null, "target_device": null }, - "layers.33.attn.wo_b": { + "layers.16.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -4000,43 +3822,40 @@ "bias": null, "target_device": null }, - "layers.2.attn.wo_a": { + "mtp.0.ffn.experts.10.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -4044,43 +3863,40 @@ "bias": null, "target_device": null }, - "layers.38.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.252.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -4088,43 +3904,40 @@ "bias": null, "target_device": null }, - "layers.9.attn.wq_a": { + "mtp.0.ffn.experts.125.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -4132,7 +3945,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.91.w2": { + "layers.45.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -4176,43 +3989,40 @@ "bias": null, "target_device": null }, - "layers.37.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.311.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -4220,43 +4030,40 @@ "bias": null, "target_device": null }, - "layers.1.attn.wkv": { + "mtp.0.ffn.experts.231.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -4264,7 +4071,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.4.w2": { + "layers.48.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -4308,7 +4115,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.118.w1": { + "layers.25.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -4352,7 +4159,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.255.w2": { + "layers.38.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -4396,43 +4203,40 @@ "bias": null, "target_device": null }, - "layers.2.attn.wq_a": { + "mtp.0.ffn.experts.13.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -4440,7 +4244,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.97.w3": { + "layers.19.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -4484,7 +4288,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.153.w2": { + "layers.22.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -4528,7 +4332,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.369.w2": { + "layers.31.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -4572,7 +4376,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.152.w3": { + "layers.23.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -4616,7 +4420,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.15.w3": { + "layers.2.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -4660,43 +4464,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.287.w2": { + "mtp.0.ffn.experts.158.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -4704,43 +4505,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.135.w2": { + "mtp.0.ffn.experts.366.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -4748,7 +4546,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.371.w2": { + "layers.0.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -4792,43 +4590,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.217.w2": { + "mtp.0.ffn.experts.261.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -4836,7 +4631,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.204.w1": { + "layers.6.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -4880,43 +4675,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.62.w2": { + "mtp.0.ffn.experts.63.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -4924,43 +4716,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.243.w3": { + "mtp.0.ffn.experts.67.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -4968,7 +4757,7 @@ "bias": null, "target_device": null }, - "layers.4.ffn.shared_experts.w3": { + "layers.46.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -5012,43 +4801,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.360.w3": { + "mtp.0.ffn.experts.163.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -5056,43 +4842,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.291.w3": { + "mtp.0.ffn.experts.164.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -5100,43 +4883,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.83.w1": { + "mtp.0.ffn.experts.57.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -5144,7 +4924,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.230.w2": { + "layers.4.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -5188,43 +4968,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.257.w2": { + "mtp.0.ffn.experts.131.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -5232,43 +5009,40 @@ "bias": null, "target_device": null }, - "layers.48.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.81.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -5276,43 +5050,40 @@ "bias": null, "target_device": null }, - "layers.39.attn.wo_b": { + "mtp.0.ffn.experts.332.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -5320,43 +5091,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.296.w3": { + "mtp.0.ffn.experts.37.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -5364,43 +5132,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.38.w2": { + "mtp.0.ffn.experts.229.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -5408,43 +5173,40 @@ "bias": null, "target_device": null }, - "layers.50.attn.wo_a": { + "mtp.0.ffn.experts.215.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -5452,43 +5214,40 @@ "bias": null, "target_device": null }, - "layers.5.attn.wq_b": { + "mtp.0.ffn.experts.58.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -5496,43 +5255,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.343.w2": { + "mtp.0.ffn.experts.318.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -5540,43 +5296,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.109.w2": { + "mtp.0.ffn.experts.192.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -5584,7 +5337,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.163.w2": { + "layers.54.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -5628,43 +5381,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.74.w2": { + "mtp.0.ffn.experts.150.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -5672,43 +5422,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.159.w2": { + "mtp.0.ffn.experts.104.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -5716,7 +5463,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.157.w2": { + "layers.16.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -5760,7 +5507,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.57.w3": { + "layers.7.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -5804,7 +5551,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.312.w1": { + "layers.8.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -5848,43 +5595,40 @@ "bias": null, "target_device": null }, - "mtp.0.h_proj": { + "mtp.0.ffn.experts.199.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -5892,43 +5636,40 @@ "bias": null, "target_device": null }, - "layers.35.attn.wq_a": { + "mtp.0.ffn.experts.246.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -5936,7 +5677,7 @@ "bias": null, "target_device": null }, - "layers.48.attn.wkv": { + "layers.56.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -5980,43 +5721,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.51.w3": { + "mtp.0.ffn.experts.220.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -6024,43 +5762,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.113.w2": { + "mtp.0.ffn.experts.280.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -6068,43 +5803,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.68.w1": { + "mtp.0.ffn.experts.345.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -6112,7 +5844,7 @@ "bias": null, "target_device": null }, - "layers.36.ffn.shared_experts.w2": { + "layers.47.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -6156,43 +5888,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.46.w3": { + "mtp.0.ffn.experts.95.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -6200,43 +5929,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.330.w2": { + "mtp.0.ffn.experts.157.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -6244,7 +5970,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.277.w2": { + "layers.57.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -6288,43 +6014,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.303.w1": { + "mtp.0.ffn.experts.22.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -6332,43 +6055,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.305.w3": { + "mtp.0.ffn.experts.236.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -6376,43 +6096,40 @@ "bias": null, "target_device": null }, - "layers.35.attn.wq_b": { + "mtp.0.ffn.experts.102.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -6420,43 +6137,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.136.w1": { + "mtp.0.ffn.experts.170.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -6464,43 +6178,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.239.w1": { + "mtp.0.ffn.experts.211.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -6508,43 +6219,40 @@ "bias": null, "target_device": null }, - "layers.23.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.256.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -6552,7 +6260,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.359.w3": { + "layers.28.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -6596,43 +6304,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.192.w2": { + "mtp.0.ffn.experts.33.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -6640,43 +6345,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.134.w1": { + "mtp.0.ffn.experts.75.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -6684,7 +6386,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.189.w2": { + "layers.45.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -6728,43 +6430,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.104.w1": { + "mtp.0.ffn.experts.216.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -6772,43 +6471,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.335.w3": { + "mtp.0.ffn.experts.101.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -6816,43 +6512,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.131.w2": { + "mtp.0.ffn.experts.125.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -6860,43 +6553,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.293.w3": { + "mtp.0.ffn.experts.230.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -6904,43 +6594,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.167.w1": { + "mtp.0.ffn.experts.142.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -6948,43 +6635,40 @@ "bias": null, "target_device": null }, - "layers.60.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.315.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -6992,43 +6676,40 @@ "bias": null, "target_device": null }, - "layers.41.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.175.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -7036,43 +6717,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.287.w3": { + "mtp.0.ffn.experts.230.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -7080,43 +6758,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.306.w2": { + "mtp.0.ffn.experts.128.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -7124,43 +6799,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.252.w3": { + "mtp.0.ffn.experts.241.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -7168,43 +6840,40 @@ "bias": null, "target_device": null }, - "layers.3.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.299.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -7212,43 +6881,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.328.w2": { + "mtp.0.ffn.experts.220.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -7256,43 +6922,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.126.w3": { + "mtp.0.ffn.experts.218.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -7300,43 +6963,40 @@ "bias": null, "target_device": null }, - "layers.29.attn.wo_b": { + "mtp.0.ffn.experts.229.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -7344,43 +7004,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.15.w1": { + "mtp.0.ffn.experts.334.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -7388,43 +7045,40 @@ "bias": null, "target_device": null }, - "layers.14.attn.wo_b": { + "mtp.0.ffn.experts.35.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -7432,43 +7086,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.275.w2": { + "mtp.0.ffn.experts.82.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -7476,7 +7127,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.335.w2": { + "layers.58.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -7520,43 +7171,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.72.w1": { + "mtp.0.ffn.experts.100.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -7564,43 +7212,40 @@ "bias": null, "target_device": null }, - "layers.58.attn.wo_a": { + "mtp.0.ffn.experts.88.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -7608,7 +7253,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.221.w3": { + "layers.56.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -7652,43 +7297,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.327.w3": { + "mtp.0.ffn.experts.342.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -7696,43 +7338,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.373.w2": { + "mtp.0.ffn.experts.302.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -7740,7 +7379,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.292.w3": { + "layers.45.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -7784,43 +7423,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.171.w2": { + "mtp.0.ffn.experts.174.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -7828,7 +7464,7 @@ "bias": null, "target_device": null }, - "layers.56.attn.wq_a": { + "layers.39.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -7872,87 +7508,40 @@ "bias": null, "target_device": null }, - "layers.13.attn.wkv": { + "mtp.0.ffn.experts.273.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, - "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, - "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", - "is_scale_quant": false, - "enable_buffer_reuse": false, - "max_input_numel": 4194304 - }, - "bias": null, - "target_device": null - }, - "layers.38.ffn.shared_experts.w1": { - "input_tensors": { - "dtype": "fp8_e4m3", - "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, - "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, - "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", - "is_scale_quant": false, - "enable_buffer_reuse": false, - "max_input_numel": 4194304 - }, - "output_tensors": null, - "weight": { - "dtype": "fp8_e4m3", - "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -7960,7 +7549,7 @@ "bias": null, "target_device": null }, - "layers.19.attn.wq_a": { + "layers.24.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -8004,7 +7593,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.36.w3": { + "layers.49.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -8048,43 +7637,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.99.w2": { + "mtp.0.ffn.experts.87.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -8092,7 +7678,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.94.w3": { + "layers.42.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -8136,7 +7722,7 @@ "bias": null, "target_device": null }, - "layers.60.attn.indexer.wq_b": { + "layers.18.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -8180,7 +7766,7 @@ "bias": null, "target_device": null }, - "layers.34.ffn.shared_experts.w1": { + "layers.23.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -8224,7 +7810,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.66.w1": { + "layers.22.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -8268,43 +7854,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.128.w2": { + "mtp.0.ffn.experts.162.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -8312,43 +7895,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.73.w3": { + "mtp.0.ffn.experts.332.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -8356,43 +7936,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.132.w2": { + "mtp.0.ffn.experts.15.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -8400,7 +7977,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.363.w1": { + "layers.34.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -8444,43 +8021,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.143.w3": { + "mtp.0.ffn.experts.154.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -8488,43 +8062,40 @@ "bias": null, "target_device": null }, - "layers.35.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.154.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -8532,43 +8103,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.205.w3": { + "mtp.0.ffn.experts.376.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -8576,7 +8144,7 @@ "bias": null, "target_device": null }, - "layers.15.attn.wq_a": { + "layers.29.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -8620,7 +8188,7 @@ "bias": null, "target_device": null }, - "layers.24.attn.wkv": { + "layers.21.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -8664,7 +8232,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.53.w2": { + "layers.31.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -8708,87 +8276,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.364.w3": { + "mtp.0.ffn.experts.79.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, - "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, - "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", - "is_scale_quant": false, - "enable_buffer_reuse": false, - "max_input_numel": 4194304 - }, - "bias": null, - "target_device": null - }, - "layers.56.attn.wkv": { - "input_tensors": { - "dtype": "fp8_e4m3", - "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, - "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, - "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", - "is_scale_quant": false, - "enable_buffer_reuse": false, - "max_input_numel": 4194304 - }, - "output_tensors": null, - "weight": { - "dtype": "fp8_e4m3", - "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -8796,43 +8317,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.305.w1": { + "mtp.0.ffn.experts.36.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -8840,43 +8358,40 @@ "bias": null, "target_device": null }, - "layers.29.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.294.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -8884,43 +8399,40 @@ "bias": null, "target_device": null }, - "layers.36.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.226.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -8928,7 +8440,7 @@ "bias": null, "target_device": null }, - "layers.9.attn.wkv": { + "layers.33.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -8972,7 +8484,7 @@ "bias": null, "target_device": null }, - "layers.44.attn.wo_a": { + "layers.59.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -9016,43 +8528,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.199.w2": { + "mtp.0.ffn.experts.5.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -9060,43 +8569,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.356.w3": { + "mtp.0.ffn.experts.316.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -9104,7 +8610,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.334.w1": { + "layers.29.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -9148,7 +8654,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.43.w1": { + "layers.54.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -9192,7 +8698,7 @@ "bias": null, "target_device": null }, - "layers.22.ffn.shared_experts.w3": { + "layers.40.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -9236,43 +8742,40 @@ "bias": null, "target_device": null }, - "layers.57.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.221.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -9280,7 +8783,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.195.w3": { + "layers.20.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -9324,43 +8827,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.191.w2": { + "mtp.0.ffn.experts.193.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -9368,43 +8868,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.242.w2": { + "mtp.0.ffn.experts.296.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -9412,7 +8909,7 @@ "bias": null, "target_device": null }, - "layers.23.ffn.shared_experts.w1": { + "layers.53.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -9456,43 +8953,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.15.w2": { + "mtp.0.ffn.experts.20.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -9500,43 +8994,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.180.w2": { + "mtp.0.ffn.experts.364.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -9544,43 +9035,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.143.w1": { + "mtp.0.ffn.experts.157.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -9588,43 +9076,40 @@ "bias": null, "target_device": null }, - "layers.5.attn.wo_b": { + "mtp.0.ffn.experts.74.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -9632,7 +9117,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.58.w1": { + "layers.2.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -9676,43 +9161,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.382.w1": { + "mtp.0.ffn.experts.77.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -9720,43 +9202,40 @@ "bias": null, "target_device": null }, - "layers.0.attn.wo_a": { + "mtp.0.ffn.experts.200.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -9764,43 +9243,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.348.w2": { + "mtp.0.ffn.experts.311.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -9808,43 +9284,40 @@ "bias": null, "target_device": null }, - "layers.23.attn.wo_a": { + "mtp.0.ffn.experts.347.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -9852,43 +9325,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.196.w2": { + "mtp.0.ffn.experts.46.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -9896,43 +9366,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.314.w1": { + "mtp.0.ffn.experts.68.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -9940,43 +9407,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.176.w3": { + "mtp.0.ffn.experts.42.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -9984,7 +9448,7 @@ "bias": null, "target_device": null }, - "layers.41.attn.wkv": { + "layers.9.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -10028,43 +9492,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.307.w1": { + "mtp.0.ffn.experts.214.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -10072,43 +9533,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.28.w2": { + "mtp.0.ffn.experts.179.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -10116,43 +9574,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.13.w1": { + "mtp.0.ffn.experts.245.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -10160,7 +9615,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.277.w3": { + "layers.27.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -10204,43 +9659,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.274.w1": { + "mtp.0.ffn.experts.303.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -10248,43 +9700,40 @@ "bias": null, "target_device": null }, - "layers.53.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.239.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -10292,43 +9741,40 @@ "bias": null, "target_device": null }, - "layers.27.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.203.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -10336,43 +9782,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.230.w1": { + "mtp.0.ffn.experts.345.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -10380,43 +9823,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.346.w3": { + "mtp.0.ffn.experts.93.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -10424,43 +9864,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.115.w1": { + "mtp.0.ffn.experts.371.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -10468,43 +9905,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.313.w3": { + "mtp.0.ffn.experts.97.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -10512,43 +9946,40 @@ "bias": null, "target_device": null }, - "layers.35.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.313.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -10556,7 +9987,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.236.w1": { + "layers.12.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -10600,43 +10031,40 @@ "bias": null, "target_device": null }, - "layers.31.attn.wo_b": { + "mtp.0.ffn.experts.101.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -10644,7 +10072,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.181.w2": { + "layers.38.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -10688,43 +10116,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.155.w3": { + "mtp.0.ffn.experts.77.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -10732,7 +10157,7 @@ "bias": null, "target_device": null }, - "layers.37.attn.wkv": { + "layers.35.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -10776,43 +10201,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.264.w3": { + "mtp.0.ffn.experts.168.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -10820,43 +10242,40 @@ "bias": null, "target_device": null }, - "layers.19.attn.wq_b": { + "mtp.0.ffn.experts.276.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -10864,43 +10283,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.38.w3": { + "mtp.0.ffn.experts.56.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -10908,7 +10324,7 @@ "bias": null, "target_device": null }, - "layers.55.attn.wo_b": { + "layers.59.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -10952,7 +10368,7 @@ "bias": null, "target_device": null }, - "layers.60.attn.wo_a": { + "layers.55.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -10996,43 +10412,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.101.w1": { + "mtp.0.ffn.experts.195.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -11040,43 +10453,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.42.w2": { + "mtp.0.ffn.experts.208.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -11084,43 +10494,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.338.w3": { + "mtp.0.ffn.experts.275.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -11128,43 +10535,40 @@ "bias": null, "target_device": null }, - "layers.21.attn.wkv": { + "mtp.0.ffn.experts.60.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -11172,43 +10576,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.301.w1": { + "mtp.0.ffn.experts.53.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -11216,43 +10617,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.179.w3": { + "mtp.0.ffn.experts.195.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -11260,43 +10658,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.271.w3": { + "mtp.0.ffn.experts.340.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -11304,43 +10699,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.11.w3": { + "mtp.0.ffn.experts.38.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -11348,43 +10740,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.147.w2": { + "mtp.0.ffn.experts.314.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -11392,7 +10781,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.3.w3": { + "layers.13.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -11436,43 +10825,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.32.w2": { + "mtp.0.ffn.experts.232.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -11480,43 +10866,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.351.w1": { + "mtp.0.ffn.experts.367.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -11524,43 +10907,40 @@ "bias": null, "target_device": null }, - "layers.58.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.322.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -11568,43 +10948,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.296.w2": { + "mtp.0.ffn.experts.178.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -11612,43 +10989,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.131.w1": { + "mtp.0.ffn.experts.166.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -11656,43 +11030,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.140.w1": { + "mtp.0.ffn.experts.298.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -11700,7 +11071,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.12.w1": { + "layers.35.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -11744,43 +11115,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.160.w2": { + "mtp.0.ffn.experts.218.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -11788,7 +11156,7 @@ "bias": null, "target_device": null }, - "layers.4.ffn.shared_experts.w2": { + "layers.60.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -11832,43 +11200,40 @@ "bias": null, "target_device": null }, - "layers.1.attn.wq_b": { + "mtp.0.ffn.experts.89.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -11876,7 +11241,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.77.w3": { + "layers.45.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -11920,7 +11285,7 @@ "bias": null, "target_device": null }, - "layers.51.ffn.shared_experts.w2": { + "layers.43.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -11964,7 +11329,7 @@ "bias": null, "target_device": null }, - "layers.57.attn.wo_a": { + "layers.13.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -12008,43 +11373,40 @@ "bias": null, "target_device": null }, - "layers.20.attn.wo_b": { + "mtp.0.ffn.experts.180.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -12052,43 +11414,40 @@ "bias": null, "target_device": null }, - "layers.21.attn.wo_b": { + "mtp.0.ffn.experts.300.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -12096,43 +11455,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.350.w1": { + "mtp.0.ffn.experts.375.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -12140,43 +11496,40 @@ "bias": null, "target_device": null }, - "layers.40.attn.wq_a": { + "mtp.0.ffn.experts.357.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -12184,43 +11537,40 @@ "bias": null, "target_device": null }, - "layers.52.attn.wo_a": { + "mtp.0.ffn.experts.122.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -12228,43 +11578,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.361.w1": { + "mtp.0.ffn.experts.382.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -12272,43 +11619,40 @@ "bias": null, "target_device": null }, - "layers.7.attn.wkv": { + "mtp.0.ffn.experts.288.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -12316,43 +11660,40 @@ "bias": null, "target_device": null }, - "layers.55.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.122.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -12360,43 +11701,40 @@ "bias": null, "target_device": null }, - "layers.18.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.9.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -12404,7 +11742,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.110.w3": { + "layers.43.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -12448,43 +11786,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.339.w1": { + "mtp.0.ffn.experts.308.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -12492,7 +11827,7 @@ "bias": null, "target_device": null }, - "layers.40.attn.wq_b": { + "layers.12.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -12536,43 +11871,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.355.w3": { + "mtp.0.ffn.experts.113.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -12580,43 +11912,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.263.w1": { + "mtp.0.ffn.experts.179.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -12624,7 +11953,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.90.w3": { + "layers.19.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -12668,43 +11997,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.363.w2": { + "mtp.0.ffn.experts.247.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -12712,7 +12038,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.193.w1": { + "layers.42.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -12756,43 +12082,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.38.w1": { + "mtp.0.ffn.experts.119.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -12800,43 +12123,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.51.w1": { + "mtp.0.ffn.experts.86.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -12844,43 +12164,40 @@ "bias": null, "target_device": null }, - "layers.50.attn.wq_b": { + "mtp.0.ffn.experts.343.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -12888,43 +12205,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.260.w1": { + "mtp.0.ffn.experts.367.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -12932,43 +12246,40 @@ "bias": null, "target_device": null }, - "layers.52.attn.wq_b": { + "mtp.0.ffn.experts.318.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -12976,43 +12287,40 @@ "bias": null, "target_device": null }, - "layers.30.attn.wkv": { + "mtp.0.ffn.experts.68.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -13020,7 +12328,7 @@ "bias": null, "target_device": null }, - "layers.17.attn.wkv": { + "layers.60.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -13064,43 +12372,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.380.w2": { + "mtp.0.ffn.experts.281.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -13108,43 +12413,40 @@ "bias": null, "target_device": null }, - "layers.6.attn.wo_a": { + "mtp.0.ffn.experts.276.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -13152,43 +12454,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.47.w1": { + "mtp.0.ffn.experts.307.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -13196,43 +12495,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.270.w2": { + "mtp.0.ffn.experts.72.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -13240,43 +12536,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.280.w3": { + "mtp.0.ffn.experts.268.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -13284,7 +12577,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.349.w3": { + "layers.52.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -13328,7 +12621,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.341.w3": { + "layers.26.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -13372,43 +12665,40 @@ "bias": null, "target_device": null }, - "layers.12.attn.wq_a": { + "mtp.0.ffn.experts.76.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -13416,7 +12706,7 @@ "bias": null, "target_device": null }, - "layers.34.attn.wq_a": { + "layers.0.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -13460,7 +12750,7 @@ "bias": null, "target_device": null }, - "layers.14.attn.wq_a": { + "layers.15.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -13504,7 +12794,7 @@ "bias": null, "target_device": null }, - "layers.24.attn.wo_b": { + "layers.8.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -13548,7 +12838,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.338.w2": { + "layers.22.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -13592,43 +12882,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.103.w1": { + "mtp.0.ffn.experts.352.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -13636,43 +12923,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.315.w3": { + "mtp.0.ffn.experts.374.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -13680,7 +12964,7 @@ "bias": null, "target_device": null }, - "layers.49.attn.wkv": { + "layers.28.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -13724,43 +13008,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.170.w2": { + "mtp.0.ffn.experts.129.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -13768,43 +13049,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.119.w2": { + "mtp.0.ffn.experts.47.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -13812,43 +13090,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.314.w2": { + "mtp.0.ffn.experts.96.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -13856,7 +13131,7 @@ "bias": null, "target_device": null }, - "layers.36.attn.wq_b": { + "layers.31.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -13900,43 +13175,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.244.w3": { + "mtp.0.ffn.experts.19.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -13944,7 +13216,7 @@ "bias": null, "target_device": null }, - "layers.13.attn.wo_b": { + "layers.38.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -13988,43 +13260,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.207.w3": { + "mtp.0.ffn.experts.135.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -14032,7 +13301,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.222.w1": { + "layers.48.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -14076,7 +13345,7 @@ "bias": null, "target_device": null }, - "layers.2.ffn.shared_experts.w1": { + "layers.44.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -14120,43 +13389,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.261.w1": { + "mtp.0.ffn.experts.376.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -14164,43 +13430,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.254.w1": { + "mtp.0.ffn.experts.237.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -14208,43 +13471,40 @@ "bias": null, "target_device": null }, - "layers.34.attn.wq_b": { + "mtp.0.ffn.experts.330.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -14252,7 +13512,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.336.w2": { + "layers.9.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -14296,43 +13556,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.51.w2": { + "mtp.0.ffn.experts.101.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -14340,43 +13597,40 @@ "bias": null, "target_device": null }, - "layers.27.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.90.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -14384,43 +13638,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.63.w2": { + "mtp.0.ffn.experts.296.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -14428,43 +13679,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.209.w2": { + "mtp.0.ffn.experts.312.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -14472,43 +13720,40 @@ "bias": null, "target_device": null }, - "layers.30.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.280.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -14516,43 +13761,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.167.w2": { + "mtp.0.ffn.experts.298.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -14560,7 +13802,7 @@ "bias": null, "target_device": null }, - "layers.49.attn.wq_a": { + "layers.3.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -14604,43 +13846,40 @@ "bias": null, "target_device": null }, - "layers.28.attn.wq_b": { + "mtp.0.ffn.experts.191.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -14648,43 +13887,40 @@ "bias": null, "target_device": null }, - "layers.43.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.67.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -14692,7 +13928,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.238.w1": { + "layers.54.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -14736,43 +13972,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.116.w3": { + "mtp.0.ffn.experts.290.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -14780,43 +14013,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.144.w1": { + "mtp.0.ffn.experts.160.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -14824,43 +14054,40 @@ "bias": null, "target_device": null }, - "layers.0.attn.wq_b": { + "mtp.0.ffn.experts.42.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -14868,43 +14095,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.316.w2": { + "mtp.0.ffn.experts.308.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -14912,43 +14136,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.110.w1": { + "mtp.0.ffn.experts.130.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -14956,7 +14177,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.191.w1": { + "layers.57.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -15000,7 +14221,7 @@ "bias": null, "target_device": null }, - "layers.58.ffn.shared_experts.w1": { + "layers.30.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -15044,43 +14265,40 @@ "bias": null, "target_device": null }, - "layers.29.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.339.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -15088,43 +14306,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.41.w1": { + "mtp.0.ffn.experts.320.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -15132,43 +14347,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.24.w3": { + "mtp.0.ffn.experts.74.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -15176,43 +14388,40 @@ "bias": null, "target_device": null }, - "layers.48.attn.wq_a": { + "mtp.0.ffn.experts.291.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -15220,43 +14429,40 @@ "bias": null, "target_device": null }, - "layers.24.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.143.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -15264,7 +14470,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.284.w1": { + "layers.34.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -15308,43 +14514,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.0.w3": { + "mtp.0.ffn.experts.131.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -15352,43 +14555,40 @@ "bias": null, "target_device": null }, - "layers.15.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.342.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -15396,7 +14596,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.378.w3": { + "layers.52.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -15440,43 +14640,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.150.w2": { + "mtp.0.ffn.experts.196.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -15484,7 +14681,7 @@ "bias": null, "target_device": null }, - "layers.19.ffn.shared_experts.w2": { + "layers.0.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -15528,43 +14725,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.30.w1": { + "mtp.0.ffn.experts.368.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -15572,43 +14766,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.113.w1": { + "mtp.0.ffn.experts.22.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -15616,43 +14807,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.158.w3": { + "mtp.0.ffn.experts.124.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -15660,43 +14848,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.57.w2": { + "mtp.0.ffn.experts.179.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -15704,43 +14889,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.17.w3": { + "mtp.0.ffn.experts.251.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -15748,43 +14930,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.269.w2": { + "mtp.0.ffn.experts.151.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -15792,43 +14971,40 @@ "bias": null, "target_device": null }, - "layers.57.attn.wq_b": { + "mtp.0.ffn.experts.197.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -15836,43 +15012,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.334.w3": { + "mtp.0.ffn.experts.34.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -15880,43 +15053,40 @@ "bias": null, "target_device": null }, - "layers.2.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.267.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -15924,43 +15094,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.106.w3": { + "mtp.0.ffn.experts.197.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -15968,43 +15135,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.266.w2": { + "mtp.0.ffn.experts.40.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -16012,43 +15176,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.30.w2": { + "mtp.0.ffn.experts.29.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -16056,43 +15217,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.139.w1": { + "mtp.0.ffn.experts.203.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -16100,43 +15258,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.47.w3": { + "mtp.0.ffn.experts.130.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -16144,43 +15299,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.344.w2": { + "mtp.0.ffn.experts.221.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -16188,43 +15340,40 @@ "bias": null, "target_device": null }, - "layers.53.attn.wo_b": { + "mtp.0.ffn.experts.165.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -16232,43 +15381,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.226.w3": { + "mtp.0.ffn.experts.150.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -16276,43 +15422,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.86.w3": { + "mtp.0.ffn.experts.120.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -16320,43 +15463,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.101.w3": { + "mtp.0.ffn.experts.168.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -16364,7 +15504,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.380.w1": { + "layers.8.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -16408,7 +15548,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.295.w2": { + "layers.34.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -16452,43 +15592,40 @@ "bias": null, "target_device": null }, - "layers.42.attn.wq_a": { + "mtp.0.ffn.experts.32.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -16496,7 +15633,7 @@ "bias": null, "target_device": null }, - "layers.46.attn.wkv": { + "layers.19.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -16540,43 +15677,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.357.w2": { + "mtp.0.ffn.experts.277.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -16584,43 +15718,40 @@ "bias": null, "target_device": null }, - "layers.17.attn.wq_a": { + "mtp.0.ffn.experts.207.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -16628,7 +15759,7 @@ "bias": null, "target_device": null }, - "layers.26.attn.wo_b": { + "layers.35.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -16672,43 +15803,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.167.w3": { + "mtp.0.ffn.experts.297.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -16716,43 +15844,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.54.w2": { + "mtp.0.ffn.experts.40.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -16760,43 +15885,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.113.w3": { + "mtp.0.ffn.experts.112.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -16804,7 +15926,7 @@ "bias": null, "target_device": null }, - "layers.25.attn.wo_b": { + "layers.32.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -16848,43 +15970,40 @@ "bias": null, "target_device": null }, - "layers.58.attn.wq_a": { + "mtp.0.ffn.experts.231.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -16892,43 +16011,40 @@ "bias": null, "target_device": null }, - "layers.55.attn.wkv": { + "mtp.0.ffn.experts.214.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -16936,43 +16052,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.371.w1": { + "mtp.0.ffn.experts.69.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -16980,43 +16093,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.302.w1": { + "mtp.0.ffn.experts.165.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -17024,7 +16134,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.156.w1": { + "layers.3.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -17068,43 +16178,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.247.w1": { + "mtp.0.ffn.experts.117.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -17112,43 +16219,40 @@ "bias": null, "target_device": null }, - "layers.44.attn.wkv": { + "mtp.0.ffn.experts.341.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -17156,7 +16260,7 @@ "bias": null, "target_device": null }, - "layers.49.attn.wq_b": { + "layers.32.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -17200,7 +16304,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.234.w2": { + "mtp.0.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -17244,7 +16348,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.70.w1": { + "layers.44.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -17288,7 +16392,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.104.w3": { + "layers.22.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -17332,7 +16436,7 @@ "bias": null, "target_device": null }, - "layers.47.attn.wo_b": { + "layers.40.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -17376,43 +16480,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.268.w2": { + "mtp.0.ffn.experts.23.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -17420,43 +16521,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.232.w1": { + "mtp.0.ffn.experts.136.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -17464,7 +16562,7 @@ "bias": null, "target_device": null }, - "layers.10.attn.wq_a": { + "layers.25.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -17508,7 +16606,7 @@ "bias": null, "target_device": null }, - "layers.33.attn.wq_b": { + "layers.55.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -17552,43 +16650,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.283.w1": { + "mtp.0.ffn.experts.215.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -17596,43 +16691,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.271.w1": { + "mtp.0.ffn.experts.112.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -17640,43 +16732,40 @@ "bias": null, "target_device": null }, - "layers.34.attn.wo_b": { + "mtp.0.ffn.experts.26.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -17684,43 +16773,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.111.w1": { + "mtp.0.ffn.experts.79.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -17728,43 +16814,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.368.w3": { + "mtp.0.ffn.experts.304.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -17772,7 +16855,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.77.w1": { + "layers.56.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -17816,7 +16899,7 @@ "bias": null, "target_device": null }, - "layers.17.ffn.shared_experts.w2": { + "layers.8.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -17860,43 +16943,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.70.w3": { + "mtp.0.ffn.experts.49.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -17904,43 +16984,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.194.w1": { + "mtp.0.ffn.experts.231.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -17948,43 +17025,40 @@ "bias": null, "target_device": null }, - "layers.10.attn.wq_b": { + "mtp.0.ffn.experts.51.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -17992,7 +17066,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.87.w2": { + "layers.52.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -18036,43 +17110,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.245.w2": { + "mtp.0.ffn.experts.7.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -18080,7 +17151,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.18.w3": { + "layers.11.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -18124,43 +17195,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.276.w3": { + "mtp.0.ffn.experts.289.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -18168,7 +17236,7 @@ "bias": null, "target_device": null }, - "layers.50.ffn.shared_experts.w1": { + "layers.26.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -18212,43 +17280,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.378.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -18256,43 +17321,40 @@ "bias": null, "target_device": null }, - "layers.52.attn.wo_b": { + "mtp.0.ffn.experts.331.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -18300,7 +17362,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.29.w1": { + "layers.7.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -18344,7 +17406,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.34.w2": { + "layers.27.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -18388,7 +17450,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.208.w3": { + "layers.21.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -18432,43 +17494,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.168.w3": { + "mtp.0.ffn.experts.256.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -18476,43 +17535,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.228.w3": { + "mtp.0.ffn.experts.283.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -18520,7 +17576,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.68.w3": { + "layers.34.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -18564,7 +17620,7 @@ "bias": null, "target_device": null }, - "layers.23.attn.wq_a": { + "layers.40.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -18608,43 +17664,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.189.w1": { + "mtp.0.ffn.experts.289.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -18652,7 +17705,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.259.w3": { + "layers.8.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -18696,43 +17749,40 @@ "bias": null, "target_device": null }, - "layers.48.attn.indexer.wq_b": { + "mtp.0.ffn.experts.282.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -18740,43 +17790,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.7.w2": { + "mtp.0.ffn.experts.128.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -18784,43 +17831,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.265.w1": { + "mtp.0.ffn.experts.87.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -18828,43 +17872,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.204.w3": { + "mtp.0.ffn.experts.180.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -18872,43 +17913,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.298.w3": { + "mtp.0.ffn.experts.87.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -18916,43 +17954,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.120.w2": { + "mtp.0.ffn.experts.172.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -18960,43 +17995,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.211.w1": { + "mtp.0.ffn.experts.147.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -19004,43 +18036,40 @@ "bias": null, "target_device": null }, - "layers.60.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.2.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -19048,7 +18077,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.275.w3": { + "layers.31.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -19092,43 +18121,40 @@ "bias": null, "target_device": null }, - "layers.14.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.240.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -19136,43 +18162,40 @@ "bias": null, "target_device": null }, - "layers.56.attn.indexer.wq_b": { + "mtp.0.ffn.experts.343.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -19180,7 +18203,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.341.w2": { + "layers.3.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -19224,7 +18247,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.142.w3": { + "layers.40.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -19268,43 +18291,40 @@ "bias": null, "target_device": null }, - "layers.28.attn.indexer.wq_b": { + "mtp.0.ffn.experts.272.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -19312,43 +18332,40 @@ "bias": null, "target_device": null }, - "layers.8.attn.wo_b": { + "mtp.0.ffn.experts.160.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -19356,43 +18373,40 @@ "bias": null, "target_device": null }, - "layers.56.attn.wo_a": { + "mtp.0.ffn.experts.181.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -19400,7 +18414,7 @@ "bias": null, "target_device": null }, - "layers.30.attn.wq_a": { + "layers.1.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -19444,7 +18458,7 @@ "bias": null, "target_device": null }, - "layers.0.ffn.shared_experts.w1": { + "layers.23.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -19488,43 +18502,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.141.w1": { + "mtp.0.ffn.experts.25.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -19532,43 +18543,40 @@ "bias": null, "target_device": null }, - "layers.11.attn.wo_b": { + "mtp.0.ffn.experts.235.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -19576,7 +18584,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.136.w2": { + "layers.17.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -19620,43 +18628,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.16.w1": { + "mtp.0.ffn.experts.278.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -19664,43 +18669,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.346.w1": { + "mtp.0.ffn.experts.368.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -19708,43 +18710,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.257.w1": { + "mtp.0.ffn.experts.238.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -19752,7 +18751,7 @@ "bias": null, "target_device": null }, - "layers.31.attn.wkv": { + "layers.36.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -19796,43 +18795,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.135.w3": { + "mtp.0.ffn.experts.21.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -19840,43 +18836,40 @@ "bias": null, "target_device": null }, - "layers.2.attn.wo_b": { + "mtp.0.ffn.experts.60.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -19884,43 +18877,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.122.w3": { + "mtp.0.ffn.experts.139.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -19928,43 +18918,40 @@ "bias": null, "target_device": null }, - "layers.38.attn.indexer.wq_b": { + "mtp.0.ffn.experts.13.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -19972,7 +18959,7 @@ "bias": null, "target_device": null }, - "layers.59.attn.wq_a": { + "layers.39.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -20016,43 +19003,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.374.w3": { + "mtp.0.ffn.experts.329.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -20060,43 +19044,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.3.w1": { + "mtp.0.ffn.experts.285.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -20104,7 +19085,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.83.w3": { + "layers.35.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -20148,43 +19129,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.60.w1": { + "mtp.0.ffn.experts.267.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -20192,43 +19170,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.26.w3": { + "mtp.0.ffn.experts.370.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -20236,7 +19211,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.9.w1": { + "layers.12.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -20280,7 +19255,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.343.w1": { + "layers.54.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -20324,43 +19299,40 @@ "bias": null, "target_device": null }, - "layers.27.attn.wo_b": { + "mtp.0.ffn.experts.44.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -20368,43 +19340,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.176.w1": { + "mtp.0.ffn.experts.261.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -20412,43 +19381,40 @@ "bias": null, "target_device": null }, - "layers.16.attn.indexer.wq_b": { + "mtp.0.ffn.experts.151.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -20456,43 +19422,40 @@ "bias": null, "target_device": null }, - "layers.43.attn.wq_b": { + "mtp.0.ffn.experts.383.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -20500,7 +19463,7 @@ "bias": null, "target_device": null }, - "layers.54.ffn.shared_experts.w3": { + "layers.2.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -20544,7 +19507,7 @@ "bias": null, "target_device": null }, - "layers.6.attn.wq_a": { + "layers.36.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -20588,43 +19551,40 @@ "bias": null, "target_device": null }, - "layers.25.attn.wkv": { + "mtp.0.ffn.experts.98.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -20632,7 +19592,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.130.w1": { + "layers.32.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -20676,43 +19636,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.369.w1": { + "mtp.0.ffn.experts.372.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -20720,7 +19677,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.194.w3": { + "layers.29.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -20764,43 +19721,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.148.w2": { + "mtp.0.ffn.experts.252.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -20808,7 +19762,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.145.w2": { + "layers.46.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -20852,43 +19806,40 @@ "bias": null, "target_device": null }, - "layers.11.attn.wq_b": { + "mtp.0.ffn.experts.212.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -20896,43 +19847,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.270.w3": { + "mtp.0.ffn.experts.327.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -20940,43 +19888,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.321.w3": { + "mtp.0.ffn.experts.239.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -20984,43 +19929,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.348.w1": { + "mtp.0.ffn.experts.286.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -21028,43 +19970,40 @@ "bias": null, "target_device": null }, - "layers.29.attn.wo_a": { + "mtp.0.ffn.experts.329.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -21072,43 +20011,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.150.w1": { + "mtp.0.ffn.experts.73.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -21116,43 +20052,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.145.w1": { + "mtp.0.ffn.experts.260.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -21160,43 +20093,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.241.w3": { + "mtp.0.ffn.experts.228.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -21204,43 +20134,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.138.w3": { + "mtp.0.ffn.experts.260.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -21248,43 +20175,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.67.w1": { + "mtp.0.ffn.experts.38.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -21292,43 +20216,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.266.w3": { + "mtp.0.ffn.experts.361.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -21336,7 +20257,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.286.w1": { + "layers.13.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -21380,43 +20301,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.273.w3": { + "mtp.0.ffn.experts.265.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -21424,7 +20342,7 @@ "bias": null, "target_device": null }, - "layers.42.attn.wkv": { + "layers.5.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -21468,7 +20386,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.94.w1": { + "layers.38.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -21512,7 +20430,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.59.w3": { + "layers.40.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -21556,7 +20474,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.378.w2": { + "layers.51.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -21600,43 +20518,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.344.w3": { + "mtp.0.ffn.experts.381.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -21644,43 +20559,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.22.w3": { + "mtp.0.ffn.experts.207.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -21688,7 +20600,7 @@ "bias": null, "target_device": null }, - "layers.41.attn.wo_b": { + "layers.27.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -21732,7 +20644,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.24.w1": { + "layers.15.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -21776,43 +20688,40 @@ "bias": null, "target_device": null }, - "layers.40.attn.wo_b": { + "mtp.0.ffn.experts.11.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -21820,7 +20729,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.358.w1": { + "layers.44.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -21864,7 +20773,7 @@ "bias": null, "target_device": null }, - "layers.8.ffn.shared_experts.w3": { + "layers.45.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -21908,43 +20817,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.42.w1": { + "mtp.0.ffn.experts.76.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -21952,43 +20858,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.274.w2": { + "mtp.0.ffn.experts.95.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -21996,43 +20899,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.232.w3": { + "mtp.0.ffn.experts.49.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -22040,7 +20940,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.95.w2": { + "layers.1.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -22084,43 +20984,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.151.w1": { + "mtp.0.ffn.experts.241.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -22128,43 +21025,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.121.w3": { + "mtp.0.ffn.experts.291.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -22172,43 +21066,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.225.w3": { + "mtp.0.ffn.experts.44.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -22216,43 +21107,40 @@ "bias": null, "target_device": null }, - "layers.31.attn.wo_a": { + "mtp.0.ffn.experts.226.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -22260,43 +21148,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.123.w2": { + "mtp.0.ffn.experts.274.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -22304,43 +21189,40 @@ "bias": null, "target_device": null }, - "layers.6.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.223.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -22348,43 +21230,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.58.w2": { + "mtp.0.ffn.experts.49.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -22392,43 +21271,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.248.w3": { + "mtp.0.ffn.experts.277.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -22436,7 +21312,7 @@ "bias": null, "target_device": null }, - "layers.10.attn.indexer.wq_b": { + "layers.2.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -22480,43 +21356,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.323.w1": { + "mtp.0.ffn.experts.109.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -22524,43 +21397,40 @@ "bias": null, "target_device": null }, - "layers.18.attn.wo_b": { + "mtp.0.ffn.experts.36.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -22568,7 +21438,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.36.w2": { + "layers.55.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -22612,7 +21482,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.345.w2": { + "layers.33.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -22656,43 +21526,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.279.w3": { + "mtp.0.ffn.experts.272.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -22700,7 +21567,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.216.w3": { + "mtp.0.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -22744,43 +21611,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.379.w3": { + "mtp.0.ffn.experts.341.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, - "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, + "round_method": "half_even", + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -22788,43 +21652,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.260.w3": { + "mtp.0.ffn.experts.346.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -22832,43 +21693,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.206.w2": { + "mtp.0.ffn.experts.252.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -22876,43 +21734,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.103.w3": { + "mtp.0.ffn.experts.204.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -22920,7 +21775,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.248.w2": { + "layers.41.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -22964,43 +21819,40 @@ "bias": null, "target_device": null }, - "layers.55.attn.wq_a": { + "mtp.0.ffn.experts.246.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -23008,7 +21860,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.273.w1": { + "layers.44.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -23052,43 +21904,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.60.w3": { + "mtp.0.ffn.experts.273.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -23096,7 +21945,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.379.w2": { + "layers.11.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -23140,43 +21989,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.27.w3": { + "mtp.0.ffn.experts.379.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -23184,43 +22030,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.324.w3": { + "mtp.0.ffn.experts.224.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -23228,43 +22071,40 @@ "bias": null, "target_device": null }, - "layers.7.attn.wq_a": { + "mtp.0.ffn.experts.341.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -23272,43 +22112,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.203.w1": { + "mtp.0.ffn.experts.354.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -23316,43 +22153,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.40.w2": { + "mtp.0.ffn.experts.9.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -23360,43 +22194,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.326.w3": { + "mtp.0.ffn.experts.251.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -23404,43 +22235,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.19.w3": { + "mtp.0.ffn.experts.32.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -23448,43 +22276,40 @@ "bias": null, "target_device": null }, - "layers.9.attn.wq_b": { + "mtp.0.ffn.experts.198.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -23492,43 +22317,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.342.w2": { + "mtp.0.ffn.experts.156.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -23536,43 +22358,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.368.w2": { + "mtp.0.ffn.experts.290.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -23580,43 +22399,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.71.w3": { + "mtp.0.ffn.experts.119.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -23624,7 +22440,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.55.w2": { + "layers.34.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -23668,43 +22484,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.341.w1": { + "mtp.0.ffn.experts.20.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -23712,43 +22525,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.291.w1": { + "mtp.0.ffn.experts.278.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -23756,43 +22566,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.226.w2": { + "mtp.0.ffn.experts.162.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -23800,43 +22607,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.207.w1": { + "mtp.0.ffn.experts.92.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -23844,43 +22648,40 @@ "bias": null, "target_device": null }, - "layers.49.attn.wo_b": { + "mtp.0.ffn.experts.206.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -23888,43 +22689,40 @@ "bias": null, "target_device": null }, - "layers.42.attn.wq_b": { + "mtp.0.ffn.experts.236.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -23932,43 +22730,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.164.w2": { + "mtp.0.ffn.experts.357.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -23976,43 +22771,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.238.w2": { + "mtp.0.ffn.experts.232.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -24020,43 +22812,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.9.w3": { + "mtp.0.ffn.experts.294.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -24064,43 +22853,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.362.w2": { + "mtp.0.ffn.experts.71.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -24108,43 +22894,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.64.w3": { + "mtp.0.ffn.experts.146.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -24152,43 +22935,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.46.w2": { + "mtp.0.ffn.experts.1.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -24196,43 +22976,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.260.w2": { + "mtp.0.ffn.experts.44.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -24240,7 +23017,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.69.w3": { + "layers.60.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -24284,43 +23061,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.124.w1": { + "mtp.0.ffn.experts.259.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -24328,7 +23102,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.375.w1": { + "layers.50.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -24372,7 +23146,7 @@ "bias": null, "target_device": null }, - "layers.51.attn.wq_a": { + "layers.39.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -24416,43 +23190,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.313.w2": { + "mtp.0.ffn.experts.142.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -24460,43 +23231,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.181.w3": { + "mtp.0.ffn.experts.298.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -24504,43 +23272,40 @@ "bias": null, "target_device": null }, - "layers.19.attn.wo_a": { + "mtp.0.ffn.experts.360.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -24548,7 +23313,7 @@ "bias": null, "target_device": null }, - "layers.15.attn.wq_b": { + "layers.41.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -24592,7 +23357,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.236.w3": { + "layers.6.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -24636,7 +23401,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.52.w2": { + "layers.58.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -24680,43 +23445,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.87.w3": { + "mtp.0.ffn.experts.123.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -24724,43 +23486,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.258.w2": { + "mtp.0.ffn.experts.94.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -24768,7 +23527,7 @@ "bias": null, "target_device": null }, - "layers.21.attn.wq_a": { + "layers.58.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -24812,43 +23571,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.134.w3": { + "mtp.0.ffn.experts.314.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -24856,43 +23612,40 @@ "bias": null, "target_device": null }, - "layers.37.attn.wq_a": { + "mtp.0.ffn.experts.103.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -24900,43 +23653,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.86.w2": { + "mtp.0.ffn.experts.219.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -24944,43 +23694,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.285.w1": { + "mtp.0.ffn.experts.339.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -24988,7 +23735,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.186.w3": { + "layers.59.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -25032,43 +23779,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.41.w2": { + "mtp.0.ffn.experts.176.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -25076,43 +23820,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.252.w2": { + "mtp.0.ffn.experts.26.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -25120,43 +23861,40 @@ "bias": null, "target_device": null }, - "layers.16.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.52.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -25164,7 +23902,7 @@ "bias": null, "target_device": null }, - "layers.47.ffn.shared_experts.w1": { + "mtp.0.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -25208,43 +23946,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.269.w1": { + "mtp.0.ffn.experts.205.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -25252,43 +23987,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.203.w3": { + "mtp.0.ffn.experts.86.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -25296,43 +24028,40 @@ "bias": null, "target_device": null }, - "layers.53.attn.wkv": { + "mtp.0.ffn.experts.295.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -25340,43 +24069,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.200.w2": { + "mtp.0.ffn.experts.363.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -25384,43 +24110,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.262.w1": { + "mtp.0.ffn.experts.109.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -25428,7 +24151,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.148.w3": { + "layers.53.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -25472,7 +24195,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.286.w2": { + "layers.28.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -25516,7 +24239,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.243.w1": { + "layers.13.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -25560,43 +24283,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.19.w1": { + "mtp.0.ffn.experts.215.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -25604,7 +24324,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.187.w2": { + "layers.40.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -25648,43 +24368,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.45.w3": { + "mtp.0.ffn.experts.70.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -25692,7 +24409,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.217.w3": { + "layers.34.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -25736,43 +24453,40 @@ "bias": null, "target_device": null }, - "layers.15.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.78.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -25780,43 +24494,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.96.w3": { + "mtp.0.ffn.experts.356.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -25824,7 +24535,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.322.w3": { + "layers.60.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -25868,43 +24579,40 @@ "bias": null, "target_device": null }, - "layers.17.attn.wq_b": { + "mtp.0.ffn.experts.77.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -25912,43 +24620,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.227.w3": { + "mtp.0.ffn.experts.383.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -25956,43 +24661,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.339.w2": { + "mtp.0.ffn.experts.30.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -26000,43 +24702,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.229.w2": { + "mtp.0.ffn.experts.279.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -26044,43 +24743,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.214.w2": { + "mtp.0.ffn.experts.30.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -26088,43 +24784,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.374.w1": { + "mtp.0.ffn.experts.167.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -26132,43 +24825,40 @@ "bias": null, "target_device": null }, - "layers.11.attn.wo_a": { + "mtp.0.ffn.experts.41.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -26176,7 +24866,7 @@ "bias": null, "target_device": null }, - "layers.8.attn.wkv": { + "layers.28.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -26220,43 +24910,40 @@ "bias": null, "target_device": null }, - "layers.28.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.20.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -26264,43 +24951,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.16.w2": { + "mtp.0.ffn.experts.110.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -26308,43 +24992,40 @@ "bias": null, "target_device": null }, - "layers.58.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.257.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -26352,7 +25033,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.107.w2": { + "layers.14.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -26396,7 +25077,7 @@ "bias": null, "target_device": null }, - "layers.55.attn.wo_a": { + "layers.29.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -26440,43 +25121,40 @@ "bias": null, "target_device": null }, - "layers.30.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.173.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -26484,43 +25162,40 @@ "bias": null, "target_device": null }, - "layers.59.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.115.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -26528,7 +25203,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.299.w3": { + "layers.1.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -26572,7 +25247,7 @@ "bias": null, "target_device": null }, - "layers.60.attn.wo_b": { + "layers.54.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -26616,7 +25291,7 @@ "bias": null, "target_device": null }, - "layers.50.attn.wo_b": { + "layers.16.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -26660,43 +25335,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.342.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -26704,7 +25376,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.221.w2": { + "layers.46.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -26748,43 +25420,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.23.w2": { + "mtp.0.ffn.experts.198.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -26792,43 +25461,40 @@ "bias": null, "target_device": null }, - "layers.52.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.308.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -26836,43 +25502,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.96.w1": { + "mtp.0.ffn.experts.169.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -26880,7 +25543,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.264.w1": { + "layers.31.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -26924,43 +25587,40 @@ "bias": null, "target_device": null }, - "layers.33.attn.wo_a": { + "mtp.0.ffn.experts.19.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -26968,43 +25628,40 @@ "bias": null, "target_device": null }, - "layers.24.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.48.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -27012,7 +25669,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.181.w1": { + "layers.18.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -27056,43 +25713,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.322.w1": { + "mtp.0.ffn.experts.301.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -27100,43 +25754,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.285.w3": { + "mtp.0.ffn.experts.373.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -27144,43 +25795,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.25.w1": { + "mtp.0.ffn.experts.2.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -27188,43 +25836,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.90.w2": { + "mtp.0.ffn.experts.334.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -27232,43 +25877,40 @@ "bias": null, "target_device": null }, - "layers.19.attn.wo_b": { + "mtp.0.ffn.experts.301.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -27276,43 +25918,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.310.w1": { + "mtp.0.ffn.experts.326.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -27320,43 +25959,40 @@ "bias": null, "target_device": null }, - "layers.22.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.126.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -27364,7 +26000,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.253.w3": { + "mtp.0.h_proj": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -27408,43 +26044,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.10.w1": { + "mtp.0.ffn.experts.121.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -27452,43 +26085,40 @@ "bias": null, "target_device": null }, - "layers.45.attn.wq_b": { + "mtp.0.ffn.experts.210.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -27496,43 +26126,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.313.w1": { + "mtp.0.ffn.experts.373.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -27540,7 +26167,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.354.w2": { + "layers.57.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -27584,7 +26211,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.59.w1": { + "layers.49.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -27628,43 +26255,40 @@ "bias": null, "target_device": null }, - "layers.37.attn.wo_b": { + "mtp.0.ffn.experts.207.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -27672,7 +26296,7 @@ "bias": null, "target_device": null }, - "layers.34.ffn.shared_experts.w3": { + "layers.5.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -27716,7 +26340,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.319.w2": { + "layers.4.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -27760,7 +26384,7 @@ "bias": null, "target_device": null }, - "layers.1.ffn.shared_experts.w2": { + "layers.54.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -27804,43 +26428,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.28.w1": { + "mtp.0.ffn.experts.51.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -27848,7 +26469,7 @@ "bias": null, "target_device": null }, - "layers.13.attn.wq_a": { + "layers.36.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -27892,43 +26513,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.349.w2": { + "mtp.0.ffn.experts.216.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -27936,7 +26554,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.291.w2": { + "layers.44.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -27980,7 +26598,7 @@ "bias": null, "target_device": null }, - "layers.7.ffn.shared_experts.w1": { + "mtp.0.e_proj": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -28024,7 +26642,7 @@ "bias": null, "target_device": null }, - "layers.3.attn.wq_b": { + "layers.56.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -28068,7 +26686,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.185.w3": { + "layers.17.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -28112,7 +26730,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.61.w1": { + "layers.2.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -28156,43 +26774,40 @@ "bias": null, "target_device": null }, - "layers.45.attn.wo_a": { + "mtp.0.ffn.experts.7.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -28200,43 +26815,40 @@ "bias": null, "target_device": null }, - "layers.39.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.248.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -28244,43 +26856,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.220.w3": { + "mtp.0.ffn.experts.122.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -28288,7 +26897,7 @@ "bias": null, "target_device": null }, - "layers.7.attn.wo_a": { + "layers.38.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -28332,7 +26941,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.88.w2": { + "layers.33.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -28376,43 +26985,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.194.w2": { + "mtp.0.ffn.experts.284.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -28420,43 +27026,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.105.w2": { + "mtp.0.ffn.experts.293.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -28464,43 +27067,40 @@ "bias": null, "target_device": null }, - "layers.57.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.240.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -28508,7 +27108,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.89.w3": { + "layers.60.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -28552,7 +27152,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.221.w1": { + "layers.53.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -28596,7 +27196,7 @@ "bias": null, "target_device": null }, - "layers.33.ffn.shared_experts.w1": { + "layers.14.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -28640,7 +27240,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.355.w2": { + "layers.15.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -28684,7 +27284,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.158.w2": { + "layers.26.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -28728,43 +27328,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.289.w2": { + "mtp.0.ffn.experts.104.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -28772,43 +27369,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.196.w1": { + "mtp.0.ffn.experts.93.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -28816,43 +27410,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.290.w1": { + "mtp.0.ffn.experts.137.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -28860,43 +27451,40 @@ "bias": null, "target_device": null }, - "layers.28.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.269.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -28904,43 +27492,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.325.w1": { + "mtp.0.ffn.experts.222.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -28948,43 +27533,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.149.w2": { + "mtp.0.ffn.experts.156.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -28992,43 +27574,40 @@ "bias": null, "target_device": null }, - "layers.9.attn.wo_a": { + "mtp.0.ffn.experts.99.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -29036,7 +27615,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.76.w3": { + "layers.14.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -29080,43 +27659,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.359.w1": { + "mtp.0.ffn.experts.59.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -29124,43 +27700,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.373.w3": { + "mtp.0.ffn.experts.267.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -29168,43 +27741,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.128.w3": { + "mtp.0.ffn.experts.47.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -29212,43 +27782,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.74.w1": { + "mtp.0.ffn.experts.287.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -29256,7 +27823,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.311.w1": { + "layers.23.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -29300,43 +27867,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.2.w3": { + "mtp.0.ffn.experts.336.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -29344,43 +27908,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.378.w1": { + "mtp.0.ffn.experts.65.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -29388,43 +27949,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.153.w3": { + "mtp.0.ffn.experts.10.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -29432,7 +27990,7 @@ "bias": null, "target_device": null }, - "layers.18.attn.wq_a": { + "layers.3.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -29476,43 +28034,40 @@ "bias": null, "target_device": null }, - "layers.21.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.158.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -29520,43 +28075,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.92.w3": { + "mtp.0.ffn.experts.139.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -29564,43 +28116,40 @@ "bias": null, "target_device": null }, - "layers.41.attn.wq_b": { + "mtp.0.ffn.experts.210.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -29608,43 +28157,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.352.w2": { + "mtp.0.ffn.experts.356.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -29652,43 +28198,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.319.w1": { + "mtp.0.ffn.experts.42.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -29696,43 +28239,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.132.w1": { + "mtp.0.ffn.experts.60.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -29740,43 +28280,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.54.w3": { + "mtp.0.ffn.experts.169.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -29784,7 +28321,7 @@ "bias": null, "target_device": null }, - "layers.37.attn.wq_b": { + "layers.19.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -29828,43 +28365,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.54.w1": { + "mtp.0.ffn.experts.13.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -29872,43 +28406,40 @@ "bias": null, "target_device": null }, - "layers.4.attn.wq_b": { + "mtp.0.ffn.experts.94.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -29916,43 +28447,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.308.w2": { + "mtp.0.ffn.experts.227.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -29960,43 +28488,40 @@ "bias": null, "target_device": null }, - "layers.7.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.328.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -30004,7 +28529,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.246.w3": { + "layers.33.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -30048,43 +28573,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.84.w1": { + "mtp.0.ffn.experts.199.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -30092,43 +28614,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.372.w1": { + "mtp.0.ffn.experts.177.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -30136,43 +28655,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.268.w3": { + "mtp.0.ffn.experts.351.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -30180,7 +28696,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.72.w3": { + "layers.10.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -30224,7 +28740,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.223.w1": { + "layers.50.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -30268,43 +28784,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.179.w2": { + "mtp.0.ffn.experts.353.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -30312,7 +28825,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.39.w3": { + "layers.27.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -30356,43 +28869,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.8.w3": { + "mtp.0.ffn.experts.174.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -30400,43 +28910,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.340.w2": { + "mtp.0.ffn.experts.3.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -30444,43 +28951,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.280.w2": { + "mtp.0.ffn.experts.111.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -30488,7 +28992,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.292.w2": { + "layers.11.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -30532,43 +29036,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.354.w1": { + "mtp.0.ffn.experts.28.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -30576,43 +29077,40 @@ "bias": null, "target_device": null }, - "layers.25.attn.wq_b": { + "mtp.0.ffn.experts.202.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -30620,43 +29118,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.182.w1": { + "mtp.0.ffn.experts.230.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -30664,43 +29159,40 @@ "bias": null, "target_device": null }, - "layers.49.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.382.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -30708,43 +29200,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.92.w2": { + "mtp.0.ffn.experts.208.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -30752,43 +29241,40 @@ "bias": null, "target_device": null }, - "layers.31.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.37.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -30796,43 +29282,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.317.w3": { + "mtp.0.ffn.experts.91.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -30840,43 +29323,40 @@ "bias": null, "target_device": null }, - "layers.32.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.377.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -30884,43 +29364,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.262.w2": { + "mtp.0.ffn.experts.212.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -30928,43 +29405,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.192.w3": { + "mtp.0.ffn.experts.312.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -30972,43 +29446,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.229.w1": { + "mtp.0.ffn.experts.73.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -31016,43 +29487,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.89.w2": { + "mtp.0.ffn.experts.124.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -31060,43 +29528,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.337.w3": { + "mtp.0.ffn.experts.242.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -31104,7 +29569,7 @@ "bias": null, "target_device": null }, - "layers.30.attn.indexer.wq_b": { + "layers.55.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -31148,43 +29613,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.138.w2": { + "mtp.0.ffn.experts.175.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -31192,43 +29654,40 @@ "bias": null, "target_device": null }, - "layers.36.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.47.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -31236,43 +29695,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.279.w1": { + "mtp.0.ffn.experts.16.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -31280,43 +29736,40 @@ "bias": null, "target_device": null }, - "layers.26.attn.indexer.wq_b": { + "mtp.0.ffn.experts.228.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -31324,43 +29777,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.199.w1": { + "mtp.0.ffn.experts.369.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -31368,43 +29818,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.48.w3": { + "mtp.0.ffn.experts.374.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -31412,43 +29859,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.133.w3": { + "mtp.0.ffn.experts.312.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -31456,43 +29900,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.381.w1": { + "mtp.0.ffn.experts.105.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -31500,7 +29941,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.233.w3": { + "layers.21.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -31544,7 +29985,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.377.w1": { + "layers.26.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -31588,7 +30029,7 @@ "bias": null, "target_device": null }, - "layers.18.attn.indexer.wq_b": { + "layers.4.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -31632,43 +30073,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.43.w3": { + "mtp.0.ffn.experts.21.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -31676,43 +30114,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.294.w3": { + "mtp.0.ffn.experts.383.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -31720,43 +30155,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.255.w1": { + "mtp.0.ffn.experts.167.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -31764,7 +30196,89 @@ "bias": null, "target_device": null }, - "layers.56.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.220.w1": { + "input_tensors": { + "dtype": "fp4", + "is_dynamic": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, + "round_method": "half_even", + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", + "mx_element_dtype": null, + "observer_cls": "PerBlockMXObserver", + "is_scale_quant": false, + "enable_buffer_reuse": false, + "max_input_numel": 4194304 + }, + "output_tensors": null, + "weight": { + "dtype": "fp4", + "is_dynamic": false, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, + "round_method": "half_even", + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", + "mx_element_dtype": null, + "observer_cls": "PerBlockMXObserver", + "is_scale_quant": false, + "enable_buffer_reuse": false, + "max_input_numel": 4194304 + }, + "bias": null, + "target_device": null + }, + "mtp.0.ffn.experts.376.w2": { + "input_tensors": { + "dtype": "fp4", + "is_dynamic": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, + "round_method": "half_even", + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", + "mx_element_dtype": null, + "observer_cls": "PerBlockMXObserver", + "is_scale_quant": false, + "enable_buffer_reuse": false, + "max_input_numel": 4194304 + }, + "output_tensors": null, + "weight": { + "dtype": "fp4", + "is_dynamic": false, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, + "round_method": "half_even", + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", + "mx_element_dtype": null, + "observer_cls": "PerBlockMXObserver", + "is_scale_quant": false, + "enable_buffer_reuse": false, + "max_input_numel": 4194304 + }, + "bias": null, + "target_device": null + }, + "layers.13.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -31808,43 +30322,122 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.183.w3": { + "mtp.0.ffn.experts.361.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, + "round_method": "half_even", + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", + "mx_element_dtype": null, + "observer_cls": "PerBlockMXObserver", + "is_scale_quant": false, + "enable_buffer_reuse": false, + "max_input_numel": 4194304 + }, + "bias": null, + "target_device": null + }, + "mtp.0.ffn.experts.172.w2": { + "input_tensors": { + "dtype": "fp4", + "is_dynamic": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, + "round_method": "half_even", + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", + "mx_element_dtype": null, + "observer_cls": "PerBlockMXObserver", + "is_scale_quant": false, + "enable_buffer_reuse": false, + "max_input_numel": 4194304 + }, + "output_tensors": null, + "weight": { + "dtype": "fp4", + "is_dynamic": false, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, + "round_method": "half_even", + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", + "mx_element_dtype": null, + "observer_cls": "PerBlockMXObserver", + "is_scale_quant": false, + "enable_buffer_reuse": false, + "max_input_numel": 4194304 + }, + "bias": null, + "target_device": null + }, + "mtp.0.ffn.experts.291.w2": { + "input_tensors": { + "dtype": "fp4", + "is_dynamic": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, + "round_method": "half_even", + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", + "mx_element_dtype": null, + "observer_cls": "PerBlockMXObserver", + "is_scale_quant": false, + "enable_buffer_reuse": false, + "max_input_numel": 4194304 + }, + "output_tensors": null, + "weight": { + "dtype": "fp4", + "is_dynamic": false, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -31852,43 +30445,40 @@ "bias": null, "target_device": null }, - "layers.41.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.294.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -31896,43 +30486,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.82.w1": { + "mtp.0.ffn.experts.188.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -31940,7 +30527,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.317.w2": { + "layers.35.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -31984,43 +30571,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.170.w1": { + "mtp.0.ffn.experts.311.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -32028,7 +30612,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.200.w3": { + "layers.47.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -32072,7 +30656,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.109.w3": { + "layers.28.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -32116,7 +30700,7 @@ "bias": null, "target_device": null }, - "layers.35.attn.wo_a": { + "layers.56.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -32160,43 +30744,40 @@ "bias": null, "target_device": null }, - "layers.59.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.349.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -32204,43 +30785,40 @@ "bias": null, "target_device": null }, - "layers.30.attn.wo_a": { + "mtp.0.ffn.experts.155.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -32248,43 +30826,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.10.w3": { + "mtp.0.ffn.experts.201.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -32292,43 +30867,40 @@ "bias": null, "target_device": null }, - "layers.54.attn.wq_b": { + "mtp.0.ffn.experts.354.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -32336,7 +30908,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.64.w1": { + "layers.41.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -32380,43 +30952,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.148.w1": { + "mtp.0.ffn.experts.321.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -32424,43 +30993,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.1.w1": { + "mtp.0.ffn.experts.299.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -32468,43 +31034,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.50.w3": { + "mtp.0.ffn.experts.356.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -32512,43 +31075,40 @@ "bias": null, "target_device": null }, - "layers.54.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.128.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -32556,43 +31116,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.34.w1": { + "mtp.0.ffn.experts.219.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -32600,43 +31157,40 @@ "bias": null, "target_device": null }, - "layers.33.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.371.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -32644,43 +31198,40 @@ "bias": null, "target_device": null }, - "layers.3.attn.wo_a": { + "mtp.0.ffn.experts.304.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -32688,7 +31239,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.69.w2": { + "layers.46.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -32732,43 +31283,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.233.w1": { + "mtp.0.ffn.experts.164.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -32776,43 +31324,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.28.w3": { + "mtp.0.ffn.experts.83.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, - "enable_buffer_reuse": false, - "max_input_numel": 4194304 - }, - "output_tensors": null, - "weight": { - "dtype": "fp8_e4m3", - "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "enable_buffer_reuse": false, + "max_input_numel": 4194304 + }, + "output_tensors": null, + "weight": { + "dtype": "fp4", + "is_dynamic": false, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -32820,43 +31365,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.243.w2": { + "mtp.0.ffn.experts.113.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -32864,43 +31406,40 @@ "bias": null, "target_device": null }, - "layers.16.attn.wo_b": { + "mtp.0.ffn.experts.162.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -32908,43 +31447,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.253.w1": { + "mtp.0.ffn.experts.293.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -32952,7 +31488,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.35.w3": { + "layers.14.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -32996,43 +31532,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.325.w2": { + "mtp.0.ffn.experts.378.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -33040,43 +31573,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.12.w2": { + "mtp.0.ffn.experts.206.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -33084,43 +31614,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.100.w3": { + "mtp.0.ffn.experts.50.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -33128,43 +31655,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.20.w3": { + "mtp.0.ffn.experts.237.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -33172,43 +31696,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.137.w1": { + "mtp.0.ffn.experts.164.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -33216,43 +31737,40 @@ "bias": null, "target_device": null }, - "layers.13.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.45.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -33260,7 +31778,7 @@ "bias": null, "target_device": null }, - "layers.21.ffn.shared_experts.w3": { + "layers.45.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -33304,7 +31822,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.22.w2": { + "layers.10.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -33348,43 +31866,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.174.w3": { + "mtp.0.ffn.experts.362.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -33392,43 +31907,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.55.w1": { + "mtp.0.ffn.experts.176.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -33436,43 +31948,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.114.w2": { + "mtp.0.ffn.experts.309.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -33480,7 +31989,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.242.w3": { + "layers.11.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -33524,43 +32033,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.287.w1": { + "mtp.0.ffn.experts.310.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -33568,43 +32074,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.299.w2": { + "mtp.0.ffn.experts.292.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -33612,43 +32115,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.171.w3": { + "mtp.0.ffn.experts.43.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -33656,43 +32156,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.372.w2": { + "mtp.0.ffn.experts.185.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, - "weight": { - "dtype": "fp8_e4m3", - "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "weight": { + "dtype": "fp4", + "is_dynamic": false, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -33700,43 +32197,40 @@ "bias": null, "target_device": null }, - "layers.13.attn.wq_b": { + "mtp.0.ffn.experts.59.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -33744,43 +32238,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.376.w1": { + "mtp.0.ffn.experts.71.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -33788,43 +32279,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.142.w2": { + "mtp.0.ffn.experts.30.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -33832,43 +32320,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.333.w2": { + "mtp.0.ffn.experts.204.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -33876,43 +32361,40 @@ "bias": null, "target_device": null }, - "layers.27.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.349.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -33920,43 +32402,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.156.w3": { + "mtp.0.ffn.experts.327.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -33964,43 +32443,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.21.w2": { + "mtp.0.ffn.experts.218.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -34008,43 +32484,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.125.w3": { + "mtp.0.ffn.experts.233.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -34052,7 +32525,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.320.w3": { + "layers.1.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -34096,43 +32569,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.286.w3": { + "mtp.0.ffn.experts.40.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -34140,43 +32610,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.330.w1": { + "mtp.0.ffn.experts.322.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -34184,43 +32651,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.177.w3": { + "mtp.0.ffn.experts.352.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -34228,43 +32692,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.246.w1": { + "mtp.0.ffn.experts.106.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -34272,7 +32733,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.228.w2": { + "layers.38.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -34316,43 +32777,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.98.w1": { + "mtp.0.ffn.experts.109.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -34360,43 +32818,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.177.w1": { + "mtp.0.ffn.experts.127.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -34404,43 +32859,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.254.w3": { + "mtp.0.ffn.experts.159.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -34448,43 +32900,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.157.w1": { + "mtp.0.ffn.experts.52.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -34492,7 +32941,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.304.w2": { + "layers.19.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -34536,7 +32985,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.102.w1": { + "layers.14.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -34580,43 +33029,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.65.w1": { + "mtp.0.ffn.experts.157.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -34624,7 +33070,7 @@ "bias": null, "target_device": null }, - "layers.42.attn.wo_b": { + "layers.10.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -34668,43 +33114,40 @@ "bias": null, "target_device": null }, - "layers.17.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.281.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -34712,7 +33155,7 @@ "bias": null, "target_device": null }, - "layers.23.attn.wq_b": { + "layers.22.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -34756,43 +33199,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.141.w2": { + "mtp.0.ffn.experts.171.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -34800,43 +33240,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.123.w1": { + "mtp.0.ffn.experts.229.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -34844,43 +33281,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.232.w2": { + "mtp.0.ffn.experts.369.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -34888,43 +33322,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.377.w3": { + "mtp.0.ffn.experts.53.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -34932,43 +33363,40 @@ "bias": null, "target_device": null }, - "layers.10.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.158.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -34976,7 +33404,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.173.w2": { + "layers.43.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -35020,43 +33448,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.288.w1": { + "mtp.0.ffn.experts.93.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -35064,43 +33489,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.224.w1": { + "mtp.0.ffn.experts.184.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -35108,43 +33530,40 @@ "bias": null, "target_device": null }, - "layers.56.attn.wo_b": { + "mtp.0.ffn.experts.173.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -35152,43 +33571,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.85.w3": { + "mtp.0.ffn.experts.269.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -35196,43 +33612,40 @@ "bias": null, "target_device": null }, - "layers.29.attn.wq_a": { + "mtp.0.ffn.experts.319.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -35240,7 +33653,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.227.w2": { + "layers.48.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -35284,7 +33697,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.216.w1": { + "layers.58.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -35328,43 +33741,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.40.w3": { + "mtp.0.ffn.experts.309.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -35372,7 +33782,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.71.w1": { + "layers.9.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -35416,7 +33826,7 @@ "bias": null, "target_device": null }, - "layers.42.ffn.shared_experts.w3": { + "layers.32.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -35460,43 +33870,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.165.w1": { + "mtp.0.ffn.experts.224.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -35504,43 +33911,40 @@ "bias": null, "target_device": null }, - "layers.3.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.148.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -35548,43 +33952,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.258.w1": { + "mtp.0.ffn.experts.184.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, - "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, + "round_method": "half_even", + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -35592,43 +33993,40 @@ "bias": null, "target_device": null }, - "layers.48.attn.wo_b": { + "mtp.0.ffn.experts.127.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -35636,43 +34034,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.296.w1": { + "mtp.0.ffn.experts.290.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -35680,43 +34075,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.219.w1": { + "mtp.0.ffn.experts.33.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -35724,43 +34116,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.197.w1": { + "mtp.0.ffn.experts.284.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -35768,43 +34157,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.317.w1": { + "mtp.0.ffn.experts.370.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -35812,7 +34198,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.297.w2": { + "layers.1.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -35856,43 +34242,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.50.w1": { + "mtp.0.ffn.experts.377.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -35900,7 +34283,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.352.w3": { + "layers.27.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -35944,7 +34327,7 @@ "bias": null, "target_device": null }, - "layers.0.ffn.shared_experts.w2": { + "layers.42.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -35988,43 +34371,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.252.w1": { + "mtp.0.ffn.experts.345.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -36032,43 +34412,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.210.w1": { + "mtp.0.ffn.experts.326.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -36076,43 +34453,40 @@ "bias": null, "target_device": null }, - "layers.8.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.6.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -36120,43 +34494,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.282.w3": { + "mtp.0.ffn.experts.134.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -36164,43 +34535,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.130.w2": { + "mtp.0.ffn.experts.300.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -36208,43 +34576,40 @@ "bias": null, "target_device": null }, - "layers.46.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.81.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -36252,7 +34617,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.309.w2": { + "layers.40.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -36296,7 +34661,7 @@ "bias": null, "target_device": null }, - "layers.8.attn.wq_b": { + "layers.5.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -36340,43 +34705,40 @@ "bias": null, "target_device": null }, - "layers.6.attn.indexer.wq_b": { + "mtp.0.ffn.experts.326.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -36384,7 +34746,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.250.w3": { + "layers.53.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -36428,43 +34790,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.155.w1": { + "mtp.0.ffn.experts.141.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -36472,7 +34831,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.215.w2": { + "layers.23.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -36516,43 +34875,40 @@ "bias": null, "target_device": null }, - "layers.1.attn.wq_a": { + "mtp.0.ffn.experts.141.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -36560,43 +34916,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.241.w2": { + "mtp.0.ffn.experts.24.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -36604,43 +34957,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.62.w1": { + "mtp.0.ffn.experts.234.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -36648,43 +34998,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.19.w2": { + "mtp.0.ffn.experts.117.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -36692,43 +35039,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.26.w2": { + "mtp.0.ffn.experts.227.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -36736,7 +35080,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.361.w2": { + "layers.49.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -36780,7 +35124,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.195.w1": { + "layers.46.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -36824,43 +35168,40 @@ "bias": null, "target_device": null }, - "layers.48.attn.wq_b": { + "mtp.0.ffn.experts.167.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -36868,7 +35209,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.285.w2": { + "layers.42.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -36912,7 +35253,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.97.w2": { + "layers.42.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -36956,43 +35297,40 @@ "bias": null, "target_device": null }, - "layers.5.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.111.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -37000,43 +35338,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.334.w2": { + "mtp.0.ffn.experts.152.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -37044,43 +35379,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.86.w1": { + "mtp.0.ffn.experts.217.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -37088,43 +35420,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.33.w1": { + "mtp.0.ffn.experts.74.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -37132,43 +35461,40 @@ "bias": null, "target_device": null }, - "layers.44.attn.wq_a": { + "mtp.0.ffn.experts.228.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -37176,43 +35502,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.92.w1": { + "mtp.0.ffn.experts.54.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -37220,43 +35543,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.193.w3": { + "mtp.0.ffn.experts.334.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -37264,43 +35584,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.264.w2": { + "mtp.0.ffn.experts.222.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -37308,43 +35625,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.149.w1": { + "mtp.0.ffn.experts.21.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -37352,7 +35666,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.119.w3": { + "layers.3.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -37396,43 +35710,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.304.w3": { + "mtp.0.ffn.experts.18.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -37440,7 +35751,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.169.w2": { + "layers.27.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -37484,43 +35795,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.128.w1": { + "mtp.0.ffn.experts.323.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -37528,7 +35836,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.74.w3": { + "layers.2.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -37572,43 +35880,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.82.w3": { + "mtp.0.ffn.experts.116.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -37616,43 +35921,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.117.w3": { + "mtp.0.ffn.experts.182.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -37660,7 +35962,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.190.w1": { + "layers.43.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -37704,43 +36006,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.70.w2": { + "mtp.0.ffn.experts.119.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -37748,43 +36047,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.174.w2": { + "mtp.0.ffn.experts.182.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -37792,43 +36088,40 @@ "bias": null, "target_device": null }, - "layers.34.attn.wkv": { + "mtp.0.ffn.experts.201.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -37836,43 +36129,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.189.w3": { + "mtp.0.ffn.experts.111.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -37880,43 +36170,40 @@ "bias": null, "target_device": null }, - "layers.58.attn.wo_b": { + "mtp.0.ffn.experts.103.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -37924,43 +36211,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.47.w2": { + "mtp.0.ffn.experts.332.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -37968,43 +36252,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.144.w3": { + "mtp.0.ffn.experts.118.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -38012,43 +36293,40 @@ "bias": null, "target_device": null }, - "layers.45.attn.wq_a": { + "mtp.0.ffn.experts.329.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -38056,43 +36334,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.80.w1": { + "mtp.0.ffn.experts.279.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -38100,43 +36375,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.255.w3": { + "mtp.0.ffn.experts.65.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -38144,43 +36416,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.188.w1": { + "mtp.0.ffn.experts.163.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -38188,43 +36457,40 @@ "bias": null, "target_device": null }, - "layers.27.attn.wq_b": { + "mtp.0.ffn.experts.67.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -38232,7 +36498,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.6.w3": { + "layers.59.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -38276,43 +36542,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.93.w2": { + "mtp.0.ffn.experts.172.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -38320,7 +36583,7 @@ "bias": null, "target_device": null }, - "layers.5.attn.wq_a": { + "layers.33.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -38364,7 +36627,7 @@ "bias": null, "target_device": null }, - "layers.15.attn.wo_b": { + "layers.50.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -38408,43 +36671,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.333.w1": { + "mtp.0.ffn.experts.238.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -38452,7 +36712,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.375.w2": { + "layers.47.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -38496,43 +36756,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.175.w1": { + "mtp.0.ffn.experts.149.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -38540,7 +36797,7 @@ "bias": null, "target_device": null }, - "layers.20.attn.wo_a": { + "layers.6.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -38584,7 +36841,7 @@ "bias": null, "target_device": null }, - "layers.25.ffn.shared_experts.w1": { + "layers.30.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -38628,7 +36885,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.158.w1": { + "layers.52.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -38672,7 +36929,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.204.w2": { + "layers.16.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -38716,7 +36973,7 @@ "bias": null, "target_device": null }, - "layers.45.attn.wkv": { + "layers.4.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -38760,7 +37017,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.217.w1": { + "layers.14.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -38804,43 +37061,40 @@ "bias": null, "target_device": null }, - "mtp.0.attn.wo_a": { + "mtp.0.ffn.experts.368.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -38848,43 +37102,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.0.w1": { + "mtp.0.ffn.experts.317.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -38892,43 +37143,40 @@ "bias": null, "target_device": null }, - "layers.5.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.212.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -38936,43 +37184,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.183.w1": { + "mtp.0.ffn.experts.317.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -38980,43 +37225,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.69.w1": { + "mtp.0.ffn.experts.39.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -39024,43 +37266,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.165.w2": { + "mtp.0.ffn.experts.275.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -39068,43 +37307,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.133.w1": { + "mtp.0.ffn.experts.262.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -39112,7 +37348,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.354.w3": { + "layers.56.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -39156,7 +37392,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.315.w1": { + "layers.48.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -39200,43 +37436,40 @@ "bias": null, "target_device": null }, - "layers.42.attn.wo_a": { + "mtp.0.ffn.experts.32.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -39244,7 +37477,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.345.w3": { + "layers.43.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -39288,43 +37521,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.117.w1": { + "mtp.0.ffn.experts.344.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 - }, - "output_tensors": null, - "weight": { - "dtype": "fp8_e4m3", - "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + }, + "output_tensors": null, + "weight": { + "dtype": "fp4", + "is_dynamic": false, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -39332,43 +37562,40 @@ "bias": null, "target_device": null }, - "layers.36.attn.wkv": { + "mtp.0.ffn.experts.200.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -39376,7 +37603,7 @@ "bias": null, "target_device": null }, - "layers.9.ffn.shared_experts.w2": { + "layers.40.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -39420,43 +37647,40 @@ "bias": null, "target_device": null }, - "layers.50.attn.wkv": { + "mtp.0.ffn.experts.265.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -39464,43 +37688,40 @@ "bias": null, "target_device": null }, - "layers.26.attn.wkv": { + "mtp.0.ffn.experts.235.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -39508,43 +37729,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.77.w2": { + "mtp.0.ffn.experts.226.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -39552,43 +37770,40 @@ "bias": null, "target_device": null }, - "layers.24.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.362.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -39596,43 +37811,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.63.w1": { + "mtp.0.ffn.experts.198.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -39640,43 +37852,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.289.w3": { + "mtp.0.ffn.experts.306.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -39684,43 +37893,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.367.w3": { + "mtp.0.ffn.experts.380.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -39728,43 +37934,40 @@ "bias": null, "target_device": null }, - "layers.16.attn.wkv": { + "mtp.0.ffn.experts.182.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -39772,43 +37975,40 @@ "bias": null, "target_device": null }, - "layers.53.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.365.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -39816,43 +38016,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.166.w1": { + "mtp.0.ffn.experts.225.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -39860,7 +38057,7 @@ "bias": null, "target_device": null }, - "layers.44.ffn.shared_experts.w3": { + "layers.47.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -39904,43 +38101,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.10.w2": { + "mtp.0.ffn.experts.335.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -39948,43 +38142,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.198.w1": { + "mtp.0.ffn.experts.140.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -39992,7 +38183,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.172.w1": { + "layers.48.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -40036,7 +38227,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.299.w1": { + "layers.48.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -40080,43 +38271,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.105.w1": { + "mtp.0.ffn.experts.208.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -40124,7 +38312,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.185.w2": { + "layers.7.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -40168,43 +38356,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.275.w1": { + "mtp.0.ffn.experts.25.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -40212,43 +38397,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.218.w1": { + "mtp.0.ffn.experts.213.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", - "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "dtype": "fp4", + "is_dynamic": false, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -40256,43 +38438,40 @@ "bias": null, "target_device": null }, - "layers.38.attn.wo_b": { + "mtp.0.ffn.experts.333.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -40300,7 +38479,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.214.w3": { + "layers.51.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -40344,43 +38523,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.308.w3": { + "mtp.0.ffn.experts.95.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -40388,43 +38564,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.172.w2": { + "mtp.0.ffn.experts.50.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -40432,43 +38605,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.5.w2": { + "mtp.0.ffn.experts.339.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -40476,43 +38646,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.162.w2": { + "mtp.0.ffn.experts.103.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -40520,43 +38687,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.180.w3": { + "mtp.0.ffn.experts.37.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -40564,43 +38728,40 @@ "bias": null, "target_device": null }, - "layers.6.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.100.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -40608,7 +38769,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.60.w2": { + "layers.43.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -40652,43 +38813,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.280.w1": { + "mtp.0.ffn.experts.130.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -40696,43 +38854,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.21.w3": { + "mtp.0.ffn.experts.211.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -40740,7 +38895,7 @@ "bias": null, "target_device": null }, - "layers.47.ffn.shared_experts.w2": { + "layers.12.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -40784,43 +38939,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.176.w2": { + "mtp.0.ffn.experts.324.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -40828,43 +38980,40 @@ "bias": null, "target_device": null }, - "layers.21.attn.wo_a": { + "mtp.0.ffn.experts.51.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -40872,43 +39021,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.265.w3": { + "mtp.0.ffn.experts.99.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -40916,43 +39062,40 @@ "bias": null, "target_device": null }, - "layers.23.attn.wkv": { + "mtp.0.ffn.experts.281.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -40960,7 +39103,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.321.w1": { + "layers.51.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -41004,43 +39147,40 @@ "bias": null, "target_device": null }, - "layers.53.attn.wq_a": { + "mtp.0.ffn.experts.16.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -41048,7 +39188,7 @@ "bias": null, "target_device": null }, - "layers.34.ffn.shared_experts.w2": { + "layers.18.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -41092,7 +39232,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.174.w1": { + "layers.57.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -41136,43 +39276,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.244.w2": { + "mtp.0.ffn.experts.344.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -41180,43 +39317,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.374.w2": { + "mtp.0.ffn.experts.12.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 - }, - "output_tensors": null, - "weight": { - "dtype": "fp8_e4m3", - "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + }, + "output_tensors": null, + "weight": { + "dtype": "fp4", + "is_dynamic": false, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -41224,43 +39358,40 @@ "bias": null, "target_device": null }, - "layers.20.attn.wq_a": { + "mtp.0.ffn.experts.63.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -41268,43 +39399,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.87.w1": { + "mtp.0.ffn.experts.189.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -41312,43 +39440,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.361.w3": { + "mtp.0.ffn.experts.117.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -41356,43 +39481,40 @@ "bias": null, "target_device": null }, - "layers.2.attn.wq_b": { + "mtp.0.ffn.experts.359.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -41400,7 +39522,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.295.w3": { + "layers.19.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -41444,7 +39566,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.202.w3": { + "layers.47.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -41488,43 +39610,40 @@ "bias": null, "target_device": null }, - "layers.3.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.155.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -41532,43 +39651,40 @@ "bias": null, "target_device": null }, - "layers.4.attn.wq_a": { + "mtp.0.ffn.experts.96.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -41576,43 +39692,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.195.w2": { + "mtp.0.ffn.experts.262.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -41620,43 +39733,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.238.w3": { + "mtp.0.ffn.experts.8.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -41664,43 +39774,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.124.w2": { + "mtp.0.ffn.experts.55.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -41708,43 +39815,40 @@ "bias": null, "target_device": null }, - "layers.32.attn.wo_b": { + "mtp.0.ffn.experts.282.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -41752,43 +39856,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.259.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -41796,43 +39897,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.350.w3": { + "mtp.0.ffn.experts.225.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -41840,43 +39938,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.99.w3": { + "mtp.0.ffn.experts.185.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -41884,43 +39979,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.159.w1": { + "mtp.0.ffn.experts.263.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -41928,43 +40020,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.329.w3": { + "mtp.0.ffn.experts.381.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -41972,43 +40061,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.277.w1": { + "mtp.0.ffn.experts.14.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, - "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, + "round_method": "half_even", + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -42016,43 +40102,40 @@ "bias": null, "target_device": null }, - "layers.53.attn.wo_a": { + "mtp.0.ffn.experts.251.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -42060,43 +40143,40 @@ "bias": null, "target_device": null }, - "layers.22.attn.wkv": { + "mtp.0.ffn.experts.239.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -42104,43 +40184,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.183.w2": { + "mtp.0.ffn.experts.132.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -42148,7 +40225,7 @@ "bias": null, "target_device": null }, - "layers.32.attn.indexer.wq_b": { + "layers.36.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -42192,7 +40269,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.309.w3": { + "layers.43.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -42236,7 +40313,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.111.w2": { + "layers.33.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -42280,43 +40357,40 @@ "bias": null, "target_device": null }, - "layers.35.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.319.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -42324,7 +40398,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.337.w1": { + "layers.31.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -42368,43 +40442,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.44.w1": { + "mtp.0.ffn.experts.305.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -42412,7 +40483,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.130.w3": { + "layers.32.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -42456,43 +40527,40 @@ "bias": null, "target_device": null }, - "layers.46.attn.wo_b": { + "mtp.0.ffn.experts.170.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -42500,7 +40568,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.153.w1": { + "layers.28.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -42544,43 +40612,40 @@ "bias": null, "target_device": null }, - "layers.43.attn.wq_a": { + "mtp.0.ffn.experts.307.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -42588,43 +40653,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.129.w2": { + "mtp.0.ffn.experts.186.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -42632,43 +40694,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.6.w1": { + "mtp.0.ffn.experts.143.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -42676,43 +40735,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.145.w3": { + "mtp.0.ffn.experts.248.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -42720,7 +40776,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.336.w1": { + "layers.50.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -42764,43 +40820,40 @@ "bias": null, "target_device": null }, - "layers.38.attn.wq_b": { + "mtp.0.ffn.experts.5.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -42808,43 +40861,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.71.w2": { + "mtp.0.ffn.experts.272.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -42852,43 +40902,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.107.w3": { + "mtp.0.ffn.experts.160.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -42896,43 +40943,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.164.w3": { + "mtp.0.ffn.experts.293.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -42940,43 +40984,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.360.w1": { + "mtp.0.ffn.experts.271.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -42984,43 +41025,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.48.w1": { + "mtp.0.ffn.experts.70.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, - "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, + "round_method": "half_even", + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -43028,7 +41066,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.144.w2": { + "layers.2.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -43072,43 +41110,40 @@ "bias": null, "target_device": null }, - "layers.32.attn.wq_b": { + "mtp.0.ffn.experts.102.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -43116,43 +41151,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.265.w2": { + "mtp.0.ffn.experts.380.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -43160,43 +41192,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.122.w2": { + "mtp.0.ffn.experts.163.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -43204,43 +41233,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.23.w1": { + "mtp.0.ffn.experts.188.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -43248,43 +41274,40 @@ "bias": null, "target_device": null }, - "layers.22.attn.wo_a": { + "mtp.0.ffn.experts.17.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -43292,43 +41315,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.151.w2": { + "mtp.0.ffn.experts.1.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -43336,7 +41356,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.376.w3": { + "layers.10.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -43380,43 +41400,40 @@ "bias": null, "target_device": null }, - "layers.40.attn.indexer.wq_b": { + "mtp.0.ffn.experts.197.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -43424,43 +41441,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.98.w3": { + "mtp.0.ffn.experts.18.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -43468,43 +41482,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.178.w3": { + "mtp.0.ffn.experts.318.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -43512,7 +41523,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.249.w2": { + "layers.16.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -43556,7 +41567,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.42.w3": { + "layers.37.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -43600,43 +41611,40 @@ "bias": null, "target_device": null }, - "layers.51.attn.wo_b": { + "mtp.0.ffn.experts.353.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -43644,43 +41652,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.24.w2": { + "mtp.0.ffn.experts.89.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -43688,43 +41693,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.278.w3": { + "mtp.0.ffn.experts.253.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -43732,43 +41734,40 @@ "bias": null, "target_device": null }, - "layers.54.attn.wq_a": { + "mtp.0.ffn.experts.1.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -43776,43 +41775,40 @@ "bias": null, "target_device": null }, - "layers.45.attn.wo_b": { + "mtp.0.ffn.experts.138.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -43820,43 +41816,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.259.w1": { + "mtp.0.ffn.experts.225.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -43864,43 +41857,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.139.w2": { + "mtp.0.ffn.experts.86.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -43908,43 +41898,40 @@ "bias": null, "target_device": null }, - "layers.18.attn.wq_b": { + "mtp.0.ffn.experts.63.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -43952,7 +41939,7 @@ "bias": null, "target_device": null }, - "layers.31.ffn.shared_experts.w2": { + "layers.0.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -43996,7 +41983,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.290.w3": { + "layers.26.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -44040,43 +42027,40 @@ "bias": null, "target_device": null }, - "layers.60.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.348.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -44084,7 +42068,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.32.w1": { + "layers.6.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -44128,43 +42112,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.331.w2": { + "mtp.0.ffn.experts.92.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -44172,43 +42153,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.283.w3": { + "mtp.0.ffn.experts.327.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -44216,7 +42194,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.4.w3": { + "layers.15.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -44260,43 +42238,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.318.w3": { + "mtp.0.ffn.experts.275.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -44304,43 +42279,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.369.w3": { + "mtp.0.ffn.experts.193.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -44348,43 +42320,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.288.w3": { + "mtp.0.ffn.experts.366.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -44392,43 +42361,40 @@ "bias": null, "target_device": null }, - "layers.41.attn.wq_a": { + "mtp.0.ffn.experts.263.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -44436,7 +42402,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.251.w1": { + "layers.57.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -44480,43 +42446,40 @@ "bias": null, "target_device": null }, - "layers.58.attn.wkv": { + "mtp.0.ffn.experts.26.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -44524,43 +42487,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.31.w1": { + "mtp.0.ffn.experts.189.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -44568,43 +42528,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.330.w3": { + "mtp.0.ffn.experts.52.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -44612,7 +42569,7 @@ "bias": null, "target_device": null }, - "layers.42.ffn.shared_experts.w2": { + "layers.8.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -44656,43 +42613,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.333.w3": { + "mtp.0.ffn.experts.315.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -44700,43 +42654,40 @@ "bias": null, "target_device": null }, - "layers.14.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.72.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -44744,43 +42695,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.222.w2": { + "mtp.0.ffn.experts.347.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -44788,43 +42736,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.147.w3": { + "mtp.0.ffn.experts.243.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -44832,7 +42777,7 @@ "bias": null, "target_device": null }, - "layers.49.attn.wo_a": { + "layers.11.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -44876,7 +42821,7 @@ "bias": null, "target_device": null }, - "layers.13.ffn.shared_experts.w1": { + "layers.20.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -44920,43 +42865,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.310.w3": { + "mtp.0.ffn.experts.324.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -44964,7 +42906,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.2.w2": { + "layers.44.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -45008,7 +42950,7 @@ "bias": null, "target_device": null }, - "layers.16.attn.wq_a": { + "layers.15.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -45052,7 +42994,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.202.w1": { + "layers.52.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -45096,43 +43038,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.331.w1": { + "mtp.0.ffn.experts.178.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -45140,7 +43079,7 @@ "bias": null, "target_device": null }, - "layers.43.attn.wkv": { + "layers.29.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -45184,7 +43123,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.100.w2": { + "layers.60.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -45228,7 +43167,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.225.w2": { + "layers.30.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -45272,43 +43211,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.4.w1": { + "mtp.0.ffn.experts.54.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -45316,43 +43252,40 @@ "bias": null, "target_device": null }, - "layers.12.attn.wkv": { + "mtp.0.ffn.experts.264.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -45360,43 +43293,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.159.w3": { + "mtp.0.ffn.experts.100.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -45404,43 +43334,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.335.w1": { + "mtp.0.ffn.experts.242.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -45448,43 +43375,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.49.w2": { + "mtp.0.ffn.experts.378.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -45492,7 +43416,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.288.w2": { + "layers.16.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -45536,43 +43460,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.97.w1": { + "mtp.0.ffn.experts.244.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -45580,43 +43501,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.31.w2": { + "mtp.0.ffn.experts.299.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -45624,43 +43542,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.292.w1": { + "mtp.0.ffn.experts.27.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -45668,7 +43583,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.178.w1": { + "layers.39.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -45712,7 +43627,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.223.w3": { + "layers.35.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -45756,43 +43671,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.108.w1": { + "mtp.0.ffn.experts.185.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -45800,43 +43712,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.72.w2": { + "mtp.0.ffn.experts.249.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -45844,43 +43753,40 @@ "bias": null, "target_device": null }, - "layers.19.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.273.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -45888,7 +43794,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.302.w3": { + "mtp.0.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -45932,7 +43838,7 @@ "bias": null, "target_device": null }, - "layers.25.attn.wq_a": { + "layers.50.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -45976,7 +43882,7 @@ "bias": null, "target_device": null }, - "layers.24.attn.indexer.wq_b": { + "layers.20.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -46020,43 +43926,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.281.w2": { + "mtp.0.ffn.experts.214.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -46064,43 +43967,40 @@ "bias": null, "target_device": null }, - "layers.14.attn.wo_a": { + "mtp.0.ffn.experts.245.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -46108,43 +44008,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.293.w1": { + "mtp.0.ffn.experts.177.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -46152,43 +44049,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.257.w3": { + "mtp.0.ffn.experts.148.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -46196,7 +44090,7 @@ "bias": null, "target_device": null }, - "layers.1.attn.wo_a": { + "layers.20.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -46240,7 +44134,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.357.w1": { + "layers.6.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -46284,7 +44178,7 @@ "bias": null, "target_device": null }, - "layers.35.attn.wkv": { + "layers.9.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -46328,7 +44222,7 @@ "bias": null, "target_device": null }, - "layers.57.attn.wo_b": { + "layers.18.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -46372,43 +44266,40 @@ "bias": null, "target_device": null }, - "layers.50.attn.wq_a": { + "mtp.0.ffn.experts.269.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -46416,43 +44307,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.223.w2": { + "mtp.0.ffn.experts.250.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -46460,43 +44348,40 @@ "bias": null, "target_device": null }, - "layers.53.attn.wq_b": { + "mtp.0.ffn.experts.255.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -46504,43 +44389,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.17.w2": { + "mtp.0.ffn.experts.336.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -46548,43 +44430,40 @@ "bias": null, "target_device": null }, - "layers.60.attn.wq_a": { + "mtp.0.ffn.experts.110.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -46592,7 +44471,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.129.w3": { + "layers.60.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -46636,43 +44515,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.282.w1": { + "mtp.0.ffn.experts.366.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -46680,43 +44556,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.146.w1": { + "mtp.0.ffn.experts.108.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -46724,43 +44597,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.212.w1": { + "mtp.0.ffn.experts.34.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -46768,43 +44638,40 @@ "bias": null, "target_device": null }, - "layers.55.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.278.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -46812,43 +44679,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.278.w2": { + "mtp.0.ffn.experts.9.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -46856,7 +44720,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.318.w1": { + "layers.12.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -46900,43 +44764,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.186.w2": { + "mtp.0.ffn.experts.105.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -46944,43 +44805,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.247.w3": { + "mtp.0.ffn.experts.270.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -46988,7 +44846,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.281.w3": { + "layers.32.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -47032,43 +44890,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.222.w3": { + "mtp.0.ffn.experts.137.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -47076,7 +44931,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.40.w1": { + "layers.14.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -47120,43 +44975,40 @@ "bias": null, "target_device": null }, - "layers.0.attn.wq_a": { + "mtp.0.ffn.experts.136.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -47164,43 +45016,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.39.w1": { + "mtp.0.ffn.experts.97.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -47208,43 +45057,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.12.w3": { + "mtp.0.ffn.experts.370.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -47252,43 +45098,40 @@ "bias": null, "target_device": null }, - "layers.32.attn.wo_a": { + "mtp.0.ffn.experts.191.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -47296,43 +45139,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.381.w2": { + "mtp.0.ffn.experts.108.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -47340,7 +45180,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.225.w1": { + "layers.36.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -47384,43 +45224,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.245.w3": { + "mtp.0.ffn.experts.323.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -47428,7 +45265,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.331.w3": { + "layers.21.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -47472,43 +45309,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.17.w1": { + "mtp.0.ffn.experts.313.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -47516,7 +45350,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.57.w1": { + "layers.55.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -47560,43 +45394,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.149.w3": { + "mtp.0.ffn.experts.159.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -47604,43 +45435,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.220.w1": { + "mtp.0.ffn.experts.292.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -47648,43 +45476,40 @@ "bias": null, "target_device": null }, - "layers.6.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.371.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -47692,43 +45517,40 @@ "bias": null, "target_device": null }, - "layers.20.attn.indexer.wq_b": { + "mtp.0.ffn.experts.180.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -47736,7 +45558,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.112.w3": { + "layers.51.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -47780,43 +45602,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.35.w2": { + "mtp.0.ffn.experts.331.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -47824,43 +45643,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.347.w3": { + "mtp.0.ffn.experts.31.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -47868,7 +45684,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.284.w2": { + "layers.47.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -47912,7 +45728,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.239.w3": { + "layers.39.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -47956,7 +45772,7 @@ "bias": null, "target_device": null }, - "layers.25.ffn.shared_experts.w2": { + "layers.57.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -48000,43 +45816,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.192.w1": { + "mtp.0.ffn.experts.89.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -48044,43 +45857,40 @@ "bias": null, "target_device": null }, - "layers.42.attn.indexer.wq_b": { + "mtp.0.ffn.experts.320.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -48088,43 +45898,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.18.w1": { + "mtp.0.ffn.experts.121.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -48132,43 +45939,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.49.w3": { + "mtp.0.ffn.experts.205.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -48176,7 +45980,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.338.w1": { + "layers.47.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -48220,43 +46024,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.78.w3": { + "mtp.0.ffn.experts.161.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -48264,7 +46065,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.142.w1": { + "layers.37.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -48308,43 +46109,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.88.w1": { + "mtp.0.ffn.experts.187.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -48352,7 +46150,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.26.w1": { + "layers.12.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -48396,7 +46194,7 @@ "bias": null, "target_device": null }, - "layers.11.attn.wq_a": { + "layers.5.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -48440,7 +46238,7 @@ "bias": null, "target_device": null }, - "layers.30.ffn.shared_experts.w3": { + "layers.28.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -48484,43 +46282,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.337.w2": { + "mtp.0.ffn.experts.194.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -48528,43 +46323,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.339.w3": { + "mtp.0.ffn.experts.186.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -48572,43 +46364,40 @@ "bias": null, "target_device": null }, - "layers.37.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.145.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -48616,43 +46405,40 @@ "bias": null, "target_device": null }, - "layers.36.attn.indexer.wq_b": { + "mtp.0.ffn.experts.64.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -48660,7 +46446,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.322.w2": { + "layers.20.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -48704,7 +46490,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.300.w1": { + "layers.26.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -48748,7 +46534,7 @@ "bias": null, "target_device": null }, - "layers.39.ffn.shared_experts.w2": { + "layers.17.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -48792,87 +46578,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.237.w3": { + "mtp.0.ffn.experts.213.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, - "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, - "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", - "is_scale_quant": false, - "enable_buffer_reuse": false, - "max_input_numel": 4194304 - }, - "bias": null, - "target_device": null - }, - "mtp.0.ffn.experts.104.w2": { - "input_tensors": { - "dtype": "fp8_e4m3", - "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", - "is_scale_quant": false, - "enable_buffer_reuse": false, - "max_input_numel": 4194304 - }, - "output_tensors": null, - "weight": { - "dtype": "fp8_e4m3", - "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, - "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, - "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -48880,43 +46619,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.27.w2": { + "mtp.0.ffn.experts.325.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -48924,43 +46660,40 @@ "bias": null, "target_device": null }, - "layers.46.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.224.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -48968,43 +46701,40 @@ "bias": null, "target_device": null }, - "layers.15.attn.wo_a": { + "mtp.0.ffn.experts.190.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -49012,7 +46742,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.14.w3": { + "layers.18.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -49056,7 +46786,7 @@ "bias": null, "target_device": null }, - "layers.0.attn.wo_b": { + "layers.23.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -49100,7 +46830,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.76.w2": { + "layers.24.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -49144,7 +46874,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.302.w2": { + "layers.48.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -49188,7 +46918,7 @@ "bias": null, "target_device": null }, - "layers.21.ffn.shared_experts.w2": { + "layers.42.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -49232,7 +46962,7 @@ "bias": null, "target_device": null }, - "layers.17.attn.wo_b": { + "layers.17.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -49276,7 +47006,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.231.w2": { + "layers.52.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -49320,43 +47050,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.138.w1": { + "mtp.0.ffn.experts.286.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -49364,43 +47091,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.99.w1": { + "mtp.0.ffn.experts.24.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -49408,43 +47132,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.375.w3": { + "mtp.0.ffn.experts.194.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -49452,43 +47173,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.187.w3": { + "mtp.0.ffn.experts.71.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -49496,43 +47214,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.365.w2": { + "mtp.0.ffn.experts.204.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -49540,7 +47255,7 @@ "bias": null, "target_device": null }, - "layers.46.ffn.shared_experts.w1": { + "layers.38.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -49584,43 +47299,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.85.w1": { + "mtp.0.ffn.experts.11.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -49628,43 +47340,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.224.w2": { + "mtp.0.ffn.experts.39.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -49672,43 +47381,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.328.w3": { + "mtp.0.ffn.experts.379.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -49716,43 +47422,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.25.w3": { + "mtp.0.ffn.experts.139.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -49760,7 +47463,7 @@ "bias": null, "target_device": null }, - "layers.40.ffn.shared_experts.w2": { + "layers.5.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -49804,7 +47507,7 @@ "bias": null, "target_device": null }, - "layers.18.ffn.shared_experts.w1": { + "layers.57.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -49848,43 +47551,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.160.w3": { + "mtp.0.ffn.experts.14.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -49892,43 +47592,40 @@ "bias": null, "target_device": null }, - "layers.18.attn.wkv": { + "mtp.0.ffn.experts.203.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -49936,43 +47633,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.35.w1": { + "mtp.0.ffn.experts.221.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -49980,7 +47674,7 @@ "bias": null, "target_device": null }, - "layers.51.attn.wo_a": { + "layers.53.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -50024,7 +47718,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.304.w1": { + "layers.55.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -50068,7 +47762,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.166.w3": { + "layers.56.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -50112,7 +47806,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.165.w3": { + "layers.38.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -50156,43 +47850,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.227.w1": { + "mtp.0.ffn.experts.15.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -50200,7 +47891,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.79.w1": { + "layers.30.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -50244,43 +47935,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.44.w2": { + "mtp.0.ffn.experts.260.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -50288,43 +47976,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.36.w1": { + "mtp.0.ffn.experts.340.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -50332,43 +48017,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.267.w2": { + "mtp.0.ffn.experts.161.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -50376,43 +48058,40 @@ "bias": null, "target_device": null }, - "layers.27.attn.wkv": { + "mtp.0.ffn.experts.161.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -50420,43 +48099,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.371.w3": { + "mtp.0.ffn.experts.317.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -50464,43 +48140,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.143.w2": { + "mtp.0.ffn.experts.76.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -50508,7 +48181,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.198.w3": { + "layers.17.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -50552,43 +48225,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.0.w2": { + "mtp.0.ffn.experts.250.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -50596,43 +48266,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.240.w3": { + "mtp.0.ffn.experts.244.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -50640,43 +48307,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.61.w3": { + "mtp.0.ffn.experts.18.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -50684,43 +48348,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.58.w3": { + "mtp.0.ffn.experts.306.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -50728,43 +48389,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.112.w2": { + "mtp.0.ffn.experts.25.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -50772,43 +48430,40 @@ "bias": null, "target_device": null }, - "layers.37.attn.wo_a": { + "mtp.0.ffn.experts.372.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -50816,7 +48471,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.75.w2": { + "layers.56.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -50860,43 +48515,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.239.w2": { + "mtp.0.ffn.experts.337.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -50904,7 +48556,7 @@ "bias": null, "target_device": null }, - "layers.46.attn.indexer.wq_b": { + "layers.30.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -50948,7 +48600,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.246.w2": { + "layers.10.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -50992,43 +48644,40 @@ "bias": null, "target_device": null }, - "layers.11.attn.wkv": { + "mtp.0.ffn.experts.116.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -51036,43 +48685,40 @@ "bias": null, "target_device": null }, - "layers.48.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.150.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -51080,43 +48726,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.274.w3": { + "mtp.0.ffn.experts.379.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -51124,43 +48767,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.235.w2": { + "mtp.0.ffn.experts.253.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -51168,43 +48808,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.251.w2": { + "mtp.0.ffn.experts.153.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -51212,43 +48849,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.13.w3": { + "mtp.0.ffn.experts.361.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -51256,7 +48890,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.125.w1": { + "layers.48.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -51300,7 +48934,7 @@ "bias": null, "target_device": null }, - "layers.6.attn.wkv": { + "layers.9.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -51344,7 +48978,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.141.w3": { + "layers.13.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -51388,43 +49022,40 @@ "bias": null, "target_device": null }, - "layers.19.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.330.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -51432,43 +49063,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.303.w3": { + "mtp.0.ffn.experts.6.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -51476,7 +49104,7 @@ "bias": null, "target_device": null }, - "layers.45.ffn.shared_experts.w3": { + "layers.42.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -51520,43 +49148,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.182.w3": { + "mtp.0.ffn.experts.292.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -51564,43 +49189,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.68.w2": { + "mtp.0.ffn.experts.81.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -51608,43 +49230,40 @@ "bias": null, "target_device": null }, - "layers.16.attn.wq_b": { + "mtp.0.ffn.experts.201.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -51652,43 +49271,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.147.w1": { + "mtp.0.ffn.experts.183.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -51696,7 +49312,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.294.w2": { + "layers.8.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -51740,43 +49356,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.353.w3": { + "mtp.0.ffn.experts.140.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -51784,43 +49397,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.219.w2": { + "mtp.0.ffn.experts.102.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -51828,43 +49438,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.168.w1": { + "mtp.0.ffn.experts.324.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -51872,7 +49479,7 @@ "bias": null, "target_device": null }, - "mtp.0.attn.wkv": { + "layers.25.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -51916,7 +49523,7 @@ "bias": null, "target_device": null }, - "layers.14.attn.wq_b": { + "layers.34.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -51960,43 +49567,40 @@ "bias": null, "target_device": null }, - "layers.54.attn.indexer.wq_b": { + "mtp.0.ffn.experts.173.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -52004,43 +49608,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.146.w3": { + "mtp.0.ffn.experts.320.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -52048,43 +49649,40 @@ "bias": null, "target_device": null }, - "layers.26.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.323.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -52092,7 +49690,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.342.w3": { + "layers.24.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -52136,43 +49734,40 @@ "bias": null, "target_device": null }, - "layers.26.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.120.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -52180,43 +49775,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.62.w3": { + "mtp.0.ffn.experts.115.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -52224,7 +49816,7 @@ "bias": null, "target_device": null }, - "layers.20.ffn.shared_experts.w3": { + "layers.18.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -52268,7 +49860,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.32.w3": { + "layers.37.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -52312,43 +49904,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.48.w2": { + "mtp.0.ffn.experts.58.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -52356,7 +49945,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.117.w2": { + "layers.49.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -52400,43 +49989,40 @@ "bias": null, "target_device": null }, - "mtp.0.attn.wq_a": { + "mtp.0.ffn.experts.12.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -52444,43 +50030,40 @@ "bias": null, "target_device": null }, - "layers.38.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.23.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -52488,43 +50071,40 @@ "bias": null, "target_device": null }, - "layers.47.attn.wo_a": { + "mtp.0.ffn.experts.277.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -52532,7 +50112,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.8.w2": { + "layers.50.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -52576,43 +50156,40 @@ "bias": null, "target_device": null }, - "layers.57.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.258.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -52620,43 +50197,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.52.w1": { + "mtp.0.ffn.experts.45.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -52664,43 +50238,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.210.w3": { + "mtp.0.ffn.experts.69.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -52708,7 +50279,7 @@ "bias": null, "target_device": null }, - "layers.27.attn.wo_a": { + "layers.6.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -52752,43 +50323,40 @@ "bias": null, "target_device": null }, - "layers.5.attn.wo_a": { + "mtp.0.ffn.experts.31.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -52796,43 +50364,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.37.w3": { + "mtp.0.ffn.experts.348.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -52840,7 +50405,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.108.w3": { + "layers.21.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -52884,43 +50449,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.102.w3": { + "mtp.0.ffn.experts.133.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -52928,7 +50490,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.121.w2": { + "layers.58.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -52972,7 +50534,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.272.w3": { + "layers.24.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -53016,7 +50578,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.270.w1": { + "layers.51.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -53060,43 +50622,40 @@ "bias": null, "target_device": null }, - "layers.58.attn.indexer.wq_b": { + "mtp.0.ffn.experts.174.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -53104,43 +50663,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.126.w2": { + "mtp.0.ffn.experts.184.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -53148,43 +50704,40 @@ "bias": null, "target_device": null }, - "layers.25.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.270.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -53192,43 +50745,40 @@ "bias": null, "target_device": null }, - "layers.32.attn.wkv": { + "mtp.0.ffn.experts.34.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -53236,43 +50786,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.162.w1": { + "mtp.0.ffn.experts.105.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -53280,43 +50827,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.272.w1": { + "mtp.0.ffn.experts.300.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -53324,43 +50868,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.364.w1": { + "mtp.0.ffn.experts.120.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -53368,43 +50909,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.237.w1": { + "mtp.0.ffn.experts.256.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -53412,43 +50950,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.23.w3": { + "mtp.0.ffn.experts.374.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -53456,43 +50991,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.210.w2": { + "mtp.0.ffn.experts.338.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -53500,43 +51032,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.76.w1": { + "mtp.0.ffn.experts.210.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, - "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, + "round_method": "half_even", + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -53544,43 +51073,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.139.w3": { + "mtp.0.ffn.experts.219.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -53588,43 +51114,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.298.w2": { + "mtp.0.ffn.experts.303.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -53632,7 +51155,7 @@ "bias": null, "target_device": null }, - "layers.46.attn.wq_a": { + "layers.46.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -53676,7 +51199,7 @@ "bias": null, "target_device": null }, - "layers.11.ffn.shared_experts.w3": { + "mtp.0.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -53720,43 +51243,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.198.w2": { + "mtp.0.ffn.experts.48.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -53764,43 +51284,40 @@ "bias": null, "target_device": null }, - "layers.9.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.380.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -53808,43 +51325,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.134.w2": { + "mtp.0.ffn.experts.285.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -53852,43 +51366,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.55.w3": { + "mtp.0.ffn.experts.195.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -53896,43 +51407,40 @@ "bias": null, "target_device": null }, - "layers.10.attn.wo_b": { + "mtp.0.ffn.experts.27.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -53940,43 +51448,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.84.w3": { + "mtp.0.ffn.experts.365.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -53984,43 +51489,40 @@ "bias": null, "target_device": null }, - "layers.44.attn.wq_b": { + "mtp.0.ffn.experts.363.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -54028,43 +51530,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.14.w1": { + "mtp.0.ffn.experts.151.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -54072,43 +51571,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.105.w3": { + "mtp.0.ffn.experts.177.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -54116,7 +51612,7 @@ "bias": null, "target_device": null }, - "layers.26.attn.wo_a": { + "layers.12.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -54160,43 +51656,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.75.w1": { + "mtp.0.ffn.experts.288.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -54204,7 +51697,7 @@ "bias": null, "target_device": null }, - "layers.28.ffn.shared_experts.w2": { + "layers.53.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -54248,43 +51741,40 @@ "bias": null, "target_device": null }, - "layers.12.attn.wo_b": { + "mtp.0.ffn.experts.90.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -54292,7 +51782,7 @@ "bias": null, "target_device": null }, - "layers.4.attn.wkv": { + "layers.58.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -54336,43 +51826,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.140.w2": { + "mtp.0.ffn.experts.266.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -54380,43 +51867,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.190.w3": { + "mtp.0.ffn.experts.247.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -54424,7 +51908,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.233.w2": { + "layers.59.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -54468,7 +51952,7 @@ "bias": null, "target_device": null }, - "layers.23.attn.wo_b": { + "layers.59.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -54512,43 +51996,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.211.w3": { + "mtp.0.ffn.experts.17.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -54556,43 +52037,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.184.w2": { + "mtp.0.ffn.experts.257.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -54600,43 +52078,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.115.w2": { + "mtp.0.ffn.experts.5.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -54644,43 +52119,40 @@ "bias": null, "target_device": null }, - "layers.8.attn.indexer.wq_b": { + "mtp.0.ffn.experts.333.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -54688,43 +52160,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.212.w3": { + "mtp.0.ffn.experts.68.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -54732,7 +52201,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.201.w1": { + "layers.26.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -54776,43 +52245,40 @@ "bias": null, "target_device": null }, - "layers.59.attn.wo_a": { + "mtp.0.ffn.experts.255.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -54820,43 +52286,40 @@ "bias": null, "target_device": null }, - "layers.15.attn.wkv": { + "mtp.0.ffn.experts.148.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -54864,7 +52327,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.258.w3": { + "layers.15.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -54908,43 +52371,40 @@ "bias": null, "target_device": null }, - "layers.22.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.41.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -54952,43 +52412,40 @@ "bias": null, "target_device": null }, - "layers.22.attn.wq_a": { + "mtp.0.ffn.experts.360.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -54996,43 +52453,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.106.w2": { + "mtp.0.ffn.experts.129.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -55040,43 +52494,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.53.w1": { + "mtp.0.ffn.experts.113.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -55084,7 +52535,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.368.w1": { + "layers.53.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -55128,43 +52579,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.370.w1": { + "mtp.0.ffn.experts.137.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -55172,43 +52620,40 @@ "bias": null, "target_device": null }, - "layers.37.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.56.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -55216,43 +52661,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.294.w1": { + "mtp.0.ffn.experts.75.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -55260,43 +52702,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.301.w2": { + "mtp.0.ffn.experts.249.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -55304,7 +52743,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.81.w1": { + "layers.33.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -55348,7 +52787,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.173.w3": { + "layers.39.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -55392,43 +52831,40 @@ "bias": null, "target_device": null }, - "layers.27.attn.wq_a": { + "mtp.0.ffn.experts.153.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -55436,43 +52872,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.323.w2": { + "mtp.0.ffn.experts.343.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -55480,7 +52913,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.327.w1": { + "layers.2.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -55524,7 +52957,7 @@ "bias": null, "target_device": null }, - "layers.0.ffn.shared_experts.w3": { + "layers.2.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -55568,7 +53001,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.269.w3": { + "layers.59.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -55612,87 +53045,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.213.w3": { + "mtp.0.ffn.experts.355.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, - "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, - "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", - "is_scale_quant": false, - "enable_buffer_reuse": false, - "max_input_numel": 4194304 - }, - "bias": null, - "target_device": null - }, - "mtp.0.ffn.experts.78.w1": { - "input_tensors": { - "dtype": "fp8_e4m3", - "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", - "is_scale_quant": false, - "enable_buffer_reuse": false, - "max_input_numel": 4194304 - }, - "output_tensors": null, - "weight": { - "dtype": "fp8_e4m3", - "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, - "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, - "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -55700,7 +53086,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.213.w1": { + "mtp.0.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -55744,7 +53130,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.372.w3": { + "layers.0.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -55788,7 +53174,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.73.w2": { + "layers.10.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -55832,7 +53218,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.364.w2": { + "layers.21.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -55876,7 +53262,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.164.w1": { + "layers.17.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -55920,7 +53306,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.160.w1": { + "layers.22.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -55964,43 +53350,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.356.w2": { + "mtp.0.ffn.experts.145.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -56008,43 +53391,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.108.w2": { + "mtp.0.ffn.experts.97.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -56052,43 +53432,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.381.w3": { + "mtp.0.ffn.experts.65.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -56096,43 +53473,40 @@ "bias": null, "target_device": null }, - "layers.40.attn.wo_a": { + "mtp.0.ffn.experts.202.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -56140,43 +53514,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.182.w2": { + "mtp.0.ffn.experts.127.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -56184,43 +53555,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.262.w3": { + "mtp.0.ffn.experts.369.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -56228,7 +53596,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.206.w1": { + "layers.41.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -56272,43 +53640,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.211.w2": { + "mtp.0.ffn.experts.46.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -56316,43 +53681,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.237.w2": { + "mtp.0.ffn.experts.135.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -56360,7 +53722,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.310.w2": { + "layers.11.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -56404,7 +53766,7 @@ "bias": null, "target_device": null }, - "layers.59.attn.wkv": { + "layers.40.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -56448,43 +53810,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.212.w2": { + "mtp.0.ffn.experts.362.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -56492,7 +53851,7 @@ "bias": null, "target_device": null }, - "layers.44.attn.indexer.wq_b": { + "layers.25.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -56536,43 +53895,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.30.w3": { + "mtp.0.ffn.experts.309.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -56580,43 +53936,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.256.w1": { + "mtp.0.ffn.experts.213.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -56624,43 +53977,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.56.w2": { + "mtp.0.ffn.experts.246.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -56668,43 +54018,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.14.w2": { + "mtp.0.ffn.experts.254.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -56712,7 +54059,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.178.w2": { + "layers.44.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -56756,7 +54103,7 @@ "bias": null, "target_device": null }, - "layers.34.attn.indexer.wq_b": { + "layers.5.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -56800,43 +54147,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.46.w1": { + "mtp.0.ffn.experts.270.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -56844,43 +54188,40 @@ "bias": null, "target_device": null }, - "layers.43.attn.wo_a": { + "mtp.0.ffn.experts.187.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -56888,43 +54229,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.6.w2": { + "mtp.0.ffn.experts.7.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -56932,43 +54270,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.169.w3": { + "mtp.0.ffn.experts.138.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -56976,43 +54311,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.13.w2": { + "mtp.0.ffn.experts.243.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -57020,7 +54352,7 @@ "bias": null, "target_device": null }, - "layers.14.ffn.shared_experts.w3": { + "layers.22.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -57064,43 +54396,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.34.w3": { + "mtp.0.ffn.experts.222.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -57108,7 +54437,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.157.w3": { + "mtp.0.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -57152,43 +54481,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.5.w3": { + "mtp.0.ffn.experts.205.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -57196,7 +54522,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.102.w2": { + "layers.20.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -57240,43 +54566,40 @@ "bias": null, "target_device": null }, - "layers.52.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.358.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -57284,43 +54607,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.244.w1": { + "mtp.0.ffn.experts.4.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -57328,7 +54648,7 @@ "bias": null, "target_device": null }, - "layers.49.ffn.shared_experts.w2": { + "layers.32.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -57372,43 +54692,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.172.w3": { + "mtp.0.ffn.experts.133.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -57416,43 +54733,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.256.w2": { + "mtp.0.ffn.experts.0.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -57460,43 +54774,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.184.w1": { + "mtp.0.ffn.experts.305.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -57504,43 +54815,40 @@ "bias": null, "target_device": null }, - "layers.59.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.367.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -57548,43 +54856,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.319.w3": { + "mtp.0.ffn.experts.82.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -57592,7 +54897,7 @@ "bias": null, "target_device": null }, - "layers.33.attn.wkv": { + "layers.3.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -57636,7 +54941,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.202.w2": { + "layers.29.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -57680,43 +54985,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.219.w3": { + "mtp.0.ffn.experts.69.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -57724,43 +55026,40 @@ "bias": null, "target_device": null }, - "layers.8.attn.wq_a": { + "mtp.0.ffn.experts.194.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -57768,7 +55067,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.78.w2": { + "layers.1.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -57812,43 +55111,40 @@ "bias": null, "target_device": null }, - "layers.17.attn.wo_a": { + "mtp.0.ffn.experts.245.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -57856,43 +55152,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.2.w1": { + "mtp.0.ffn.experts.363.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -57900,43 +55193,40 @@ "bias": null, "target_device": null }, - "layers.48.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.183.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -57944,43 +55234,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.180.w1": { + "mtp.0.ffn.experts.235.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -57988,43 +55275,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.240.w2": { + "mtp.0.ffn.experts.359.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -58032,43 +55316,40 @@ "bias": null, "target_device": null }, - "layers.7.attn.wo_b": { + "mtp.0.ffn.experts.115.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -58076,7 +55357,7 @@ "bias": null, "target_device": null }, - "layers.52.attn.wq_a": { + "layers.30.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -58120,43 +55401,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.111.w3": { + "mtp.0.ffn.experts.259.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -58164,43 +55442,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.45.w2": { + "mtp.0.ffn.experts.144.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -58208,7 +55483,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.307.w3": { + "layers.27.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -58252,43 +55527,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.315.w2": { + "mtp.0.ffn.experts.41.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -58296,7 +55568,7 @@ "bias": null, "target_device": null }, - "layers.55.attn.wq_b": { + "layers.53.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -58340,7 +55612,7 @@ "bias": null, "target_device": null }, - "layers.12.attn.wq_b": { + "layers.58.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -58384,7 +55656,7 @@ "bias": null, "target_device": null }, - "layers.38.attn.wo_a": { + "layers.41.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -58428,43 +55700,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.184.w3": { + "mtp.0.ffn.experts.59.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -58472,43 +55741,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.173.w1": { + "mtp.0.ffn.experts.170.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -58516,7 +55782,7 @@ "bias": null, "target_device": null }, - "layers.39.attn.wkv": { + "layers.51.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -58560,43 +55826,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.367.w2": { + "mtp.0.ffn.experts.372.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -58604,7 +55867,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.297.w1": { + "layers.18.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -58648,7 +55911,7 @@ "bias": null, "target_device": null }, - "layers.57.attn.wq_a": { + "layers.4.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -58692,43 +55955,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.376.w2": { + "mtp.0.ffn.experts.175.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -58736,43 +55996,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.348.w3": { + "mtp.0.ffn.experts.106.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -58780,43 +56037,40 @@ "bias": null, "target_device": null }, - "layers.24.attn.wo_a": { + "mtp.0.ffn.experts.274.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -58824,7 +56078,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.152.w2": { + "layers.18.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -58868,7 +56122,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.91.w3": { + "layers.12.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -58912,43 +56166,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.240.w1": { + "mtp.0.ffn.experts.303.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -58956,43 +56207,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.154.w2": { + "mtp.0.ffn.experts.143.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -59000,43 +56248,40 @@ "bias": null, "target_device": null }, - "layers.12.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.98.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -59044,43 +56289,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.123.w3": { + "mtp.0.ffn.experts.364.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -59088,43 +56330,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.324.w1": { + "mtp.0.ffn.experts.283.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -59132,7 +56371,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.259.w2": { + "layers.50.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -59164,11 +56403,52 @@ ], "symmetric": true, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float8_e8m0fnu", + "scale_format": null, + "scale_calculation_mode": null, + "mx_element_dtype": null, + "observer_cls": "PerBlock2DMinMaxObserver", + "is_scale_quant": false, + "enable_buffer_reuse": false, + "max_input_numel": 4194304 + }, + "bias": null, + "target_device": null + }, + "mtp.0.ffn.experts.82.w3": { + "input_tensors": { + "dtype": "fp4", + "is_dynamic": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, + "round_method": "half_even", + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", + "mx_element_dtype": null, + "observer_cls": "PerBlockMXObserver", + "is_scale_quant": false, + "enable_buffer_reuse": false, + "max_input_numel": 4194304 + }, + "output_tensors": null, + "weight": { + "dtype": "fp4", + "is_dynamic": false, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, + "round_method": "half_even", + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -59176,7 +56456,7 @@ "bias": null, "target_device": null }, - "layers.36.attn.wo_a": { + "layers.49.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -59220,43 +56500,40 @@ "bias": null, "target_device": null }, - "layers.4.attn.indexer.wq_b": { + "mtp.0.ffn.experts.295.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -59264,43 +56541,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.256.w3": { + "mtp.0.ffn.experts.31.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -59308,7 +56582,7 @@ "bias": null, "target_device": null }, - "layers.40.attn.wkv": { + "layers.55.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -59352,43 +56626,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.127.w2": { + "mtp.0.ffn.experts.114.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -59396,43 +56667,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.370.w2": { + "mtp.0.ffn.experts.61.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -59440,43 +56708,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.343.w3": { + "mtp.0.ffn.experts.39.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -59484,43 +56749,40 @@ "bias": null, "target_device": null }, - "layers.54.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.196.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -59528,7 +56790,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.353.w2": { + "layers.8.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -59572,43 +56834,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.185.w1": { + "mtp.0.ffn.experts.248.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -59616,43 +56875,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.205.w2": { + "mtp.0.ffn.experts.190.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -59660,43 +56916,40 @@ "bias": null, "target_device": null }, - "layers.12.attn.wo_a": { + "mtp.0.ffn.experts.301.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -59704,43 +56957,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.91.w1": { + "mtp.0.ffn.experts.337.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -59748,43 +56998,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.188.w3": { + "mtp.0.ffn.experts.118.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -59792,43 +57039,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.107.w1": { + "mtp.0.ffn.experts.192.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -59836,43 +57080,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.215.w1": { + "mtp.0.ffn.experts.29.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -59880,43 +57121,40 @@ "bias": null, "target_device": null }, - "layers.35.attn.wo_b": { + "mtp.0.ffn.experts.357.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -59924,43 +57162,40 @@ "bias": null, "target_device": null }, - "layers.47.attn.wkv": { + "mtp.0.ffn.experts.344.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -59968,7 +57203,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.283.w2": { + "layers.58.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -60012,43 +57247,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.230.w3": { + "mtp.0.ffn.experts.271.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, - "weight": { - "dtype": "fp8_e4m3", - "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "weight": { + "dtype": "fp4", + "is_dynamic": false, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -60056,43 +57288,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.350.w2": { + "mtp.0.ffn.experts.237.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -60100,43 +57329,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.209.w3": { + "mtp.0.ffn.experts.271.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -60144,7 +57370,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.120.w3": { + "layers.27.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -60188,7 +57414,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.197.w2": { + "layers.30.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -60232,43 +57458,40 @@ "bias": null, "target_device": null }, - "mtp.0.attn.wq_b": { + "mtp.0.ffn.experts.147.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -60276,43 +57499,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.329.w1": { + "mtp.0.ffn.experts.295.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -60320,43 +57540,40 @@ "bias": null, "target_device": null }, - "layers.6.attn.wo_b": { + "mtp.0.ffn.experts.114.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -60364,43 +57581,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.52.w3": { + "mtp.0.ffn.experts.131.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -60408,43 +57622,40 @@ "bias": null, "target_device": null }, - "layers.21.attn.wq_b": { + "mtp.0.ffn.experts.94.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -60452,43 +57663,40 @@ "bias": null, "target_device": null }, - "mtp.0.e_proj": { + "mtp.0.ffn.experts.325.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -60496,7 +57704,7 @@ "bias": null, "target_device": null }, - "layers.6.attn.wq_b": { + "layers.45.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -60540,43 +57748,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.53.w3": { + "mtp.0.ffn.experts.359.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -60584,7 +57789,7 @@ "bias": null, "target_device": null }, - "layers.16.ffn.shared_experts.w1": { + "layers.59.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -60628,43 +57833,40 @@ "bias": null, "target_device": null }, - "layers.17.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.353.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -60672,43 +57874,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.324.w2": { + "mtp.0.ffn.experts.289.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -60716,43 +57915,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.218.w2": { + "mtp.0.ffn.experts.142.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -60760,43 +57956,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.115.w3": { + "mtp.0.ffn.experts.335.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -60804,43 +57997,40 @@ "bias": null, "target_device": null }, - "layers.41.attn.wo_a": { + "mtp.0.ffn.experts.302.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -60848,7 +58038,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.179.w1": { + "layers.7.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -60892,7 +58082,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.306.w1": { + "layers.0.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -60936,43 +58126,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.186.w1": { + "mtp.0.ffn.experts.258.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -60980,43 +58167,40 @@ "bias": null, "target_device": null }, - "layers.44.attn.wo_b": { + "mtp.0.ffn.experts.188.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -61024,43 +58208,40 @@ "bias": null, "target_device": null }, - "layers.52.attn.indexer.wq_b": { + "mtp.0.ffn.experts.322.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -61068,43 +58249,40 @@ "bias": null, "target_device": null }, - "layers.22.attn.wq_b": { + "mtp.0.ffn.experts.264.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -61112,43 +58290,40 @@ "bias": null, "target_device": null }, - "layers.11.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.16.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -61156,7 +58331,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.101.w2": { + "layers.17.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -61200,7 +58375,7 @@ "bias": null, "target_device": null }, - "layers.51.attn.wkv": { + "layers.14.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -61244,7 +58419,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.347.w2": { + "layers.54.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -61288,43 +58463,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.314.w3": { + "mtp.0.ffn.experts.321.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -61332,7 +58504,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.325.w3": { + "layers.13.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -61376,43 +58548,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.127.w1": { + "mtp.0.ffn.experts.53.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -61420,7 +58589,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.8.w1": { + "layers.29.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -61464,43 +58633,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.65.w2": { + "mtp.0.ffn.experts.253.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -61508,43 +58674,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.345.w1": { + "mtp.0.ffn.experts.147.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -61552,43 +58715,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.29.w3": { + "mtp.0.ffn.experts.43.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -61596,7 +58756,7 @@ "bias": null, "target_device": null }, - "layers.1.attn.wo_b": { + "layers.15.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -61640,43 +58800,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.88.w3": { + "mtp.0.ffn.experts.381.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -61684,43 +58841,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.249.w3": { + "mtp.0.ffn.experts.189.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -61728,43 +58882,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.224.w3": { + "mtp.0.ffn.experts.58.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -61772,43 +58923,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.214.w1": { + "mtp.0.ffn.experts.83.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -61816,43 +58964,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.110.w2": { + "mtp.0.ffn.experts.287.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -61860,7 +59005,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.156.w2": { + "layers.32.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -61904,43 +59049,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.120.w1": { + "mtp.0.ffn.experts.187.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -61948,43 +59090,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.169.w1": { + "mtp.0.ffn.experts.233.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -61992,43 +59131,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.236.w2": { + "mtp.0.ffn.experts.91.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -62036,7 +59172,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.63.w3": { + "layers.52.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -62080,43 +59216,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.7.w1": { + "mtp.0.ffn.experts.223.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -62124,43 +59257,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.89.w1": { + "mtp.0.ffn.experts.108.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -62168,43 +59298,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.200.w1": { + "mtp.0.ffn.experts.364.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -62212,43 +59339,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.301.w3": { + "mtp.0.ffn.experts.11.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -62256,7 +59380,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.366.w2": { + "layers.9.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -62300,43 +59424,40 @@ "bias": null, "target_device": null }, - "layers.0.attn.wkv": { + "mtp.0.ffn.experts.254.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -62344,43 +59465,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.234.w3": { + "mtp.0.ffn.experts.123.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -62388,43 +59506,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.332.w3": { + "mtp.0.ffn.experts.373.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -62432,43 +59547,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.281.w1": { + "mtp.0.ffn.experts.355.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -62476,43 +59588,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.279.w2": { + "mtp.0.ffn.experts.190.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -62520,43 +59629,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.247.w2": { + "mtp.0.ffn.experts.257.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -62564,7 +59670,7 @@ "bias": null, "target_device": null }, - "layers.56.ffn.shared_experts.w3": { + "layers.55.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -62608,7 +59714,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.65.w3": { + "layers.24.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -62652,43 +59758,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.56.w3": { + "mtp.0.ffn.experts.149.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -62696,43 +59799,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.163.w1": { + "mtp.0.ffn.experts.88.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -62740,43 +59840,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.253.w2": { + "mtp.0.ffn.experts.134.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -62784,43 +59881,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.248.w1": { + "mtp.0.ffn.experts.200.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -62828,43 +59922,40 @@ "bias": null, "target_device": null }, - "layers.41.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.336.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -62872,43 +59963,40 @@ "bias": null, "target_device": null }, - "layers.51.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.43.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -62916,7 +60004,7 @@ "bias": null, "target_device": null }, - "layers.3.attn.wo_b": { + "layers.38.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -62960,7 +60048,7 @@ "bias": null, "target_device": null }, - "layers.4.attn.wo_b": { + "layers.24.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -63004,43 +60092,40 @@ "bias": null, "target_device": null }, - "layers.28.attn.wkv": { + "mtp.0.ffn.experts.240.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -63048,7 +60133,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.132.w3": { + "layers.11.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -63092,7 +60177,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.121.w1": { + "layers.19.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -63136,43 +60221,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.203.w2": { + "mtp.0.ffn.experts.124.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -63180,7 +60262,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.300.w3": { + "layers.6.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -63224,7 +60306,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.162.w3": { + "layers.57.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -63268,7 +60350,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.208.w1": { + "layers.42.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -63312,43 +60394,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.137.w3": { + "mtp.0.ffn.experts.313.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -63356,43 +60435,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.31.w3": { + "mtp.0.ffn.experts.280.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -63400,7 +60476,7 @@ "bias": null, "target_device": null }, - "layers.50.attn.indexer.wq_b": { + "layers.15.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -63444,43 +60520,40 @@ "bias": null, "target_device": null }, - "layers.59.attn.wq_b": { + "mtp.0.ffn.experts.107.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -63488,7 +60561,7 @@ "bias": null, "target_device": null }, - "layers.3.attn.wq_a": { + "layers.4.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -63532,43 +60605,40 @@ "bias": null, "target_device": null }, - "layers.29.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.138.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -63576,43 +60646,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.272.w2": { + "mtp.0.ffn.experts.2.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -63620,43 +60687,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.380.w3": { + "mtp.0.ffn.experts.351.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -63664,43 +60728,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.79.w2": { + "mtp.0.ffn.experts.305.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -63708,43 +60769,40 @@ "bias": null, "target_device": null }, - "layers.39.attn.wq_b": { + "mtp.0.ffn.experts.316.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -63752,7 +60810,7 @@ "bias": null, "target_device": null }, - "layers.7.attn.wq_b": { + "layers.37.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -63796,43 +60854,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.84.w2": { + "mtp.0.ffn.experts.78.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -63840,43 +60895,40 @@ "bias": null, "target_device": null }, - "layers.29.attn.wq_b": { + "mtp.0.ffn.experts.331.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -63884,7 +60936,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.95.w3": { + "layers.45.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -63928,43 +60980,81 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.327.w2": { + "mtp.0.ffn.experts.223.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, + "round_method": "half_even", + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", + "mx_element_dtype": null, + "observer_cls": "PerBlockMXObserver", + "is_scale_quant": false, + "enable_buffer_reuse": false, + "max_input_numel": 4194304 + }, + "bias": null, + "target_device": null + }, + "mtp.0.ffn.experts.121.w2": { + "input_tensors": { + "dtype": "fp4", + "is_dynamic": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, + "round_method": "half_even", + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", + "mx_element_dtype": null, + "observer_cls": "PerBlockMXObserver", + "is_scale_quant": false, + "enable_buffer_reuse": false, + "max_input_numel": 4194304 + }, + "output_tensors": null, + "weight": { + "dtype": "fp4", + "is_dynamic": false, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -63972,43 +61062,40 @@ "bias": null, "target_device": null }, - "layers.32.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.236.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -64016,43 +61103,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.61.w2": { + "mtp.0.ffn.experts.244.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -64060,7 +61144,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.41.w3": { + "layers.33.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -64104,43 +61188,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.27.w1": { + "mtp.0.ffn.experts.181.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -64148,43 +61229,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.340.w1": { + "mtp.0.ffn.experts.352.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -64192,7 +61270,7 @@ "bias": null, "target_device": null }, - "layers.22.attn.indexer.wq_b": { + "layers.25.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -64236,43 +61314,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.379.w1": { + "mtp.0.ffn.experts.199.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -64280,43 +61355,40 @@ "bias": null, "target_device": null }, - "layers.52.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.262.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -64324,7 +61396,7 @@ "bias": null, "target_device": null }, - "layers.16.attn.wo_a": { + "layers.20.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -64368,43 +61440,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.98.w2": { + "mtp.0.ffn.experts.3.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -64412,43 +61481,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.80.w3": { + "mtp.0.ffn.experts.19.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -64456,43 +61522,40 @@ "bias": null, "target_device": null }, - "layers.36.attn.wo_b": { + "mtp.0.ffn.experts.217.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -64500,7 +61563,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.328.w1": { + "layers.10.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -64544,7 +61607,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.114.w1": { + "layers.5.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -64588,43 +61651,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.208.w2": { + "mtp.0.ffn.experts.78.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -64632,7 +61692,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.73.w1": { + "layers.46.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -64676,43 +61736,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.155.w2": { + "mtp.0.ffn.experts.55.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -64720,43 +61777,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.103.w2": { + "mtp.0.ffn.experts.382.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -64764,43 +61818,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.175.w2": { + "mtp.0.ffn.experts.14.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -64808,43 +61859,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.190.w2": { + "mtp.0.ffn.experts.250.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -64852,43 +61900,40 @@ "bias": null, "target_device": null }, - "layers.2.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.36.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -64896,43 +61941,40 @@ "bias": null, "target_device": null }, - "layers.54.attn.wo_a": { + "mtp.0.ffn.experts.314.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", - "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "dtype": "fp4", + "is_dynamic": false, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -64940,43 +61982,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.323.w3": { + "mtp.0.ffn.experts.166.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -64984,43 +62023,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.289.w1": { + "mtp.0.ffn.experts.45.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -65028,7 +62064,7 @@ "bias": null, "target_device": null }, - "layers.2.attn.indexer.wq_b": { + "layers.23.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -65072,7 +62108,7 @@ "bias": null, "target_device": null }, - "layers.19.attn.wkv": { + "layers.10.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -65116,43 +62152,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.168.w2": { + "mtp.0.ffn.experts.206.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -65160,7 +62193,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.166.w2": { + "layers.41.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -65204,43 +62237,40 @@ "bias": null, "target_device": null }, - "layers.38.attn.wq_a": { + "mtp.0.ffn.experts.104.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -65248,43 +62278,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.342.w1": { + "mtp.0.ffn.experts.70.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -65292,43 +62319,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.357.w3": { + "mtp.0.ffn.experts.265.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -65336,43 +62360,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.377.w2": { + "mtp.0.ffn.experts.296.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -65380,43 +62401,40 @@ "bias": null, "target_device": null }, - "layers.38.attn.wkv": { + "mtp.0.ffn.experts.62.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -65424,7 +62442,7 @@ "bias": null, "target_device": null }, - "layers.8.attn.wo_a": { + "layers.8.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -65468,43 +62486,40 @@ "bias": null, "target_device": null }, - "layers.1.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.186.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -65512,43 +62527,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.154.w3": { + "mtp.0.ffn.experts.279.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -65556,43 +62568,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.151.w3": { + "mtp.0.ffn.experts.264.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -65600,43 +62609,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.188.w2": { + "mtp.0.ffn.experts.171.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -65644,43 +62650,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.362.w3": { + "mtp.0.ffn.experts.132.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -65688,43 +62691,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.56.w1": { + "mtp.0.ffn.experts.73.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -65732,43 +62732,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.16.w3": { + "mtp.0.ffn.experts.261.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -65776,43 +62773,40 @@ "bias": null, "target_device": null }, - "layers.16.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.17.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -65820,43 +62814,40 @@ "bias": null, "target_device": null }, - "layers.47.attn.wq_b": { + "mtp.0.ffn.experts.12.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -65864,43 +62855,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.161.w2": { + "mtp.0.ffn.experts.123.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -65908,7 +62896,7 @@ "bias": null, "target_device": null }, - "layers.45.ffn.shared_experts.w2": { + "layers.41.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -65952,7 +62940,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.112.w1": { + "layers.42.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -65996,43 +62984,40 @@ "bias": null, "target_device": null }, - "layers.10.attn.wkv": { + "mtp.0.ffn.experts.156.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -66040,43 +63025,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.321.w2": { + "mtp.0.ffn.experts.141.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -66084,43 +63066,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.67.w2": { + "mtp.0.ffn.experts.335.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -66128,7 +63107,7 @@ "bias": null, "target_device": null }, - "layers.43.ffn.shared_experts.w2": { + "layers.10.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -66172,43 +63151,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.241.w1": { + "mtp.0.ffn.experts.110.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -66216,43 +63192,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.346.w2": { + "mtp.0.ffn.experts.276.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -66260,43 +63233,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.193.w2": { + "mtp.0.ffn.experts.22.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -66304,43 +63274,40 @@ "bias": null, "target_device": null }, - "layers.47.attn.wq_a": { + "mtp.0.ffn.experts.116.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -66348,43 +63315,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.201.w2": { + "mtp.0.ffn.experts.84.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -66392,43 +63356,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.207.w2": { + "mtp.0.ffn.experts.126.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -66436,7 +63397,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.365.w1": { + "layers.24.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -66480,7 +63441,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.316.w1": { + "layers.21.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -66524,43 +63485,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.44.w3": { + "mtp.0.ffn.experts.181.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -66568,43 +63526,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.125.w2": { + "mtp.0.ffn.experts.96.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -66612,43 +63567,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.235.w3": { + "mtp.0.ffn.experts.48.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -66656,43 +63608,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.276.w1": { + "mtp.0.ffn.experts.211.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -66700,43 +63649,40 @@ "bias": null, "target_device": null }, - "layers.51.attn.wq_b": { + "mtp.0.ffn.experts.8.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -66744,7 +63690,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.312.w2": { + "layers.34.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -66788,7 +63734,7 @@ "bias": null, "target_device": null }, - "layers.23.ffn.shared_experts.w2": { + "layers.36.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -66832,43 +63778,40 @@ "bias": null, "target_device": null }, - "layers.54.attn.wo_b": { + "mtp.0.ffn.experts.29.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -66876,7 +63819,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.251.w3": { + "layers.28.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -66920,7 +63863,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.308.w1": { + "layers.11.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -66964,43 +63907,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.43.w2": { + "mtp.0.ffn.experts.88.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -67008,87 +63948,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.59.w2": { + "mtp.0.ffn.experts.23.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, - "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, - "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", - "is_scale_quant": false, - "enable_buffer_reuse": false, - "max_input_numel": 4194304 - }, - "bias": null, - "target_device": null - }, - "mtp.0.ffn.experts.25.w2": { - "input_tensors": { - "dtype": "fp8_e4m3", - "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", - "is_scale_quant": false, - "enable_buffer_reuse": false, - "max_input_numel": 4194304 - }, - "output_tensors": null, - "weight": { - "dtype": "fp8_e4m3", - "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, - "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, - "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -67096,43 +63989,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.309.w1": { + "mtp.0.ffn.experts.310.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -67140,7 +64030,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.250.w1": { + "layers.1.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -67184,7 +64074,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.152.w1": { + "layers.49.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -67228,43 +64118,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.311.w2": { + "mtp.0.ffn.experts.350.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -67272,7 +64159,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.90.w1": { + "layers.7.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -67316,43 +64203,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.213.w2": { + "mtp.0.ffn.experts.171.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -67360,7 +64244,7 @@ "bias": null, "target_device": null }, - "layers.10.attn.wo_a": { + "layers.37.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -67404,43 +64288,40 @@ "bias": null, "target_device": null }, - "layers.30.attn.wq_b": { + "mtp.0.ffn.experts.328.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, - "weight": { - "dtype": "fp8_e4m3", - "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "weight": { + "dtype": "fp4", + "is_dynamic": false, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -67448,7 +64329,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.33.w2": { + "layers.12.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -67492,7 +64373,7 @@ "bias": null, "target_device": null }, - "layers.59.attn.wo_b": { + "layers.46.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -67536,43 +64417,40 @@ "bias": null, "target_device": null }, - "layers.26.attn.wq_a": { + "mtp.0.ffn.experts.268.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -67580,43 +64458,40 @@ "bias": null, "target_device": null }, - "layers.60.attn.wq_b": { + "mtp.0.ffn.experts.28.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -67624,43 +64499,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.234.w1": { + "mtp.0.ffn.experts.55.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -67668,43 +64540,40 @@ "bias": null, "target_device": null }, - "layers.43.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.144.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -67712,43 +64581,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.242.w1": { + "mtp.0.ffn.experts.0.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -67756,43 +64622,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.49.w1": { + "mtp.0.ffn.experts.10.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -67800,43 +64663,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.81.w3": { + "mtp.0.ffn.experts.183.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -67844,43 +64704,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.226.w1": { + "mtp.0.ffn.experts.129.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -67888,43 +64745,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.20.w1": { + "mtp.0.ffn.experts.297.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -67932,43 +64786,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.85.w2": { + "mtp.0.ffn.experts.347.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -67976,43 +64827,40 @@ "bias": null, "target_device": null }, - "layers.57.attn.wkv": { + "mtp.0.ffn.experts.283.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -68020,43 +64868,40 @@ "bias": null, "target_device": null }, - "layers.4.attn.wo_a": { + "mtp.0.ffn.experts.136.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -68064,43 +64909,40 @@ "bias": null, "target_device": null }, - "layers.31.attn.wq_a": { + "mtp.0.ffn.experts.169.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -68108,43 +64950,40 @@ "bias": null, "target_device": null }, - "layers.50.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.145.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -68152,43 +64991,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.94.w2": { + "mtp.0.ffn.experts.287.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -68196,7 +65032,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.161.w1": { + "layers.3.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -68240,43 +65076,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.370.w3": { + "mtp.0.ffn.experts.80.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, - "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, + "round_method": "half_even", + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -68284,7 +65117,7 @@ "bias": null, "target_device": null }, - "layers.29.attn.wkv": { + "layers.54.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -68328,7 +65161,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.326.w2": { + "layers.35.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -68372,7 +65205,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.114.w3": { + "layers.60.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -68416,43 +65249,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.271.w2": { + "mtp.0.ffn.experts.274.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -68460,7 +65290,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.351.w3": { + "layers.39.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -68504,7 +65334,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.11.w2": { + "layers.24.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -68548,7 +65378,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.45.w1": { + "layers.19.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -68592,43 +65422,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.300.w2": { + "mtp.0.ffn.experts.64.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -68636,43 +65463,40 @@ "bias": null, "target_device": null }, - "layers.56.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.146.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -68680,43 +65504,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.75.w3": { + "mtp.0.ffn.experts.66.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -68724,43 +65545,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.215.w3": { + "mtp.0.ffn.experts.75.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -68768,43 +65586,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.250.w2": { + "mtp.0.ffn.experts.64.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -68812,43 +65627,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.305.w2": { + "mtp.0.ffn.experts.242.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -68856,7 +65668,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.229.w3": { + "layers.28.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -68900,43 +65712,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.358.w3": { + "mtp.0.ffn.experts.91.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -68944,7 +65753,7 @@ "bias": null, "target_device": null }, - "layers.44.ffn.shared_experts.w1": { + "layers.25.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -68988,43 +65797,40 @@ "bias": null, "target_device": null }, - "layers.20.attn.wkv": { + "mtp.0.ffn.experts.56.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -69032,43 +65838,40 @@ "bias": null, "target_device": null }, - "layers.9.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.98.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -69076,43 +65879,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.96.w2": { + "mtp.0.ffn.experts.349.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -69120,43 +65920,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.118.w3": { + "mtp.0.ffn.experts.85.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -69164,43 +65961,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.109.w1": { + "mtp.0.ffn.experts.85.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -69208,7 +66002,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.297.w3": { + "layers.52.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -69252,43 +66046,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.340.w3": { + "mtp.0.ffn.experts.238.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -69296,43 +66087,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.21.w1": { + "mtp.0.ffn.experts.35.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -69340,43 +66128,40 @@ "bias": null, "target_device": null }, - "layers.20.attn.wq_b": { + "mtp.0.ffn.experts.255.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -69384,43 +66169,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.50.w2": { + "mtp.0.ffn.experts.338.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -69428,7 +66210,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.278.w1": { + "layers.6.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -69472,43 +66254,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.383.w1": { + "mtp.0.ffn.experts.50.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -69516,43 +66295,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.196.w3": { + "mtp.0.ffn.experts.328.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -69560,7 +66336,7 @@ "bias": null, "target_device": null }, - "layers.51.ffn.shared_experts.w1": { + "layers.44.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -69604,43 +66380,40 @@ "bias": null, "target_device": null }, - "layers.58.attn.wq_b": { + "mtp.0.ffn.experts.192.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -69648,7 +66421,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.347.w1": { + "layers.20.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -69692,43 +66465,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.20.w2": { + "mtp.0.ffn.experts.66.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -69736,7 +66506,7 @@ "bias": null, "target_device": null }, - "layers.28.attn.wo_a": { + "layers.21.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -69780,43 +66550,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.163.w3": { + "mtp.0.ffn.experts.24.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -69824,7 +66591,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.124.w3": { + "layers.24.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -69868,43 +66635,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.363.w3": { + "mtp.0.ffn.experts.46.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -69912,7 +66676,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.360.w2": { + "layers.16.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -69956,7 +66720,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.93.w3": { + "layers.48.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -70000,7 +66764,7 @@ "bias": null, "target_device": null }, - "layers.26.ffn.shared_experts.w3": { + "layers.50.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -70044,43 +66808,40 @@ "bias": null, "target_device": null }, - "layers.46.attn.wq_b": { + "mtp.0.ffn.experts.232.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -70088,43 +66849,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.64.w2": { + "mtp.0.ffn.experts.360.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -70132,43 +66890,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.79.w3": { + "mtp.0.ffn.experts.340.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -70176,43 +66931,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.267.w1": { + "mtp.0.ffn.experts.3.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -70220,7 +66972,7 @@ "bias": null, "target_device": null }, - "layers.15.ffn.shared_experts.w2": { + "layers.18.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -70264,43 +67016,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.355.w1": { + "mtp.0.ffn.experts.217.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -70308,43 +67057,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.33.w3": { + "mtp.0.ffn.experts.346.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -70352,43 +67098,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.131.w3": { + "mtp.0.ffn.experts.304.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -70396,43 +67139,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.3.w2": { + "mtp.0.ffn.experts.196.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -70440,43 +67180,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.201.w3": { + "mtp.0.ffn.experts.337.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", - "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "dtype": "fp4", + "is_dynamic": false, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -70484,7 +67221,7 @@ "bias": null, "target_device": null }, - "layers.33.attn.wq_a": { + "layers.35.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -70528,43 +67265,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.18.w2": { + "mtp.0.ffn.experts.321.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -70572,43 +67306,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.263.w2": { + "mtp.0.ffn.experts.178.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -70616,43 +67347,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.383.w2": { + "mtp.0.ffn.experts.216.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -70660,43 +67388,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.1.w2": { + "mtp.0.ffn.experts.33.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -70704,43 +67429,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.249.w1": { + "mtp.0.ffn.experts.233.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -70748,43 +67470,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.175.w3": { + "mtp.0.ffn.experts.99.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -70792,7 +67511,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.66.w2": { + "layers.51.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -70836,7 +67555,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.365.w3": { + "layers.17.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -70880,43 +67599,40 @@ "bias": null, "target_device": null }, - "layers.52.attn.wkv": { + "mtp.0.ffn.experts.282.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -70924,7 +67640,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.177.w2": { + "layers.20.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -70968,43 +67684,40 @@ "bias": null, "target_device": null }, - "layers.49.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.4.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -71012,43 +67725,40 @@ "bias": null, "target_device": null }, - "layers.32.attn.wq_a": { + "mtp.0.ffn.experts.350.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -71056,43 +67766,40 @@ "bias": null, "target_device": null }, - "mtp.0.attn.wo_b": { + "mtp.0.ffn.experts.57.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -71100,43 +67807,40 @@ "bias": null, "target_device": null }, - "layers.45.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.61.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -71144,43 +67848,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.146.w2": { + "mtp.0.ffn.experts.209.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -71188,43 +67889,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.205.w1": { + "mtp.0.ffn.experts.348.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -71232,43 +67930,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.154.w1": { + "mtp.0.ffn.experts.79.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -71276,43 +67971,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.359.w2": { + "mtp.0.ffn.experts.358.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -71320,7 +68012,7 @@ "bias": null, "target_device": null }, - "layers.26.attn.wq_b": { + "layers.4.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -71364,7 +68056,7 @@ "bias": null, "target_device": null }, - "layers.24.attn.wq_b": { + "layers.51.attn.wo_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -71408,43 +68100,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.197.w3": { + "mtp.0.ffn.experts.243.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -71452,43 +68141,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.298.w1": { + "mtp.0.ffn.experts.176.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -71496,7 +68182,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.9.w2": { + "layers.32.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -71540,7 +68226,7 @@ "bias": null, "target_device": null }, - "layers.43.attn.wo_b": { + "layers.43.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -71584,43 +68270,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.235.w1": { + "mtp.0.ffn.experts.107.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -71628,43 +68311,40 @@ "bias": null, "target_device": null }, - "layers.39.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.263.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -71672,43 +68352,40 @@ "bias": null, "target_device": null }, - "layers.2.attn.wkv": { + "mtp.0.ffn.experts.107.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -71716,43 +68393,40 @@ "bias": null, "target_device": null }, - "layers.48.attn.wo_a": { + "mtp.0.ffn.experts.330.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -71760,7 +68434,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.218.w3": { + "layers.6.attn.indexer.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -71804,43 +68478,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.116.w2": { + "mtp.0.ffn.experts.92.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -71848,43 +68519,40 @@ "bias": null, "target_device": null }, - "layers.20.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.118.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -71892,43 +68560,40 @@ "bias": null, "target_device": null }, - "layers.36.attn.wq_a": { + "mtp.0.ffn.experts.315.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -71936,43 +68601,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.140.w3": { + "mtp.0.ffn.experts.57.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -71980,7 +68642,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.276.w2": { + "layers.23.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -72024,43 +68686,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.126.w1": { + "mtp.0.ffn.experts.227.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -72068,7 +68727,7 @@ "bias": null, "target_device": null }, - "layers.40.ffn.shared_experts.w1": { + "layers.9.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -72112,43 +68771,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.127.w3": { + "mtp.0.ffn.experts.84.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -72156,7 +68812,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.199.w3": { + "layers.26.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -72200,43 +68856,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.82.w2": { + "mtp.0.ffn.experts.135.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -72244,43 +68897,40 @@ "bias": null, "target_device": null }, - "layers.10.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.28.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -72288,43 +68938,40 @@ "bias": null, "target_device": null }, - "layers.12.attn.indexer.wq_b": { + "mtp.0.ffn.experts.54.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -72332,43 +68979,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.7.w3": { + "mtp.0.ffn.experts.152.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -72376,43 +69020,40 @@ "bias": null, "target_device": null }, - "layers.18.attn.wo_a": { + "mtp.0.ffn.experts.191.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, - "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, + "round_method": "half_even", + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -72420,7 +69061,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.136.w3": { + "layers.50.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -72464,43 +69105,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.5.w1": { + "mtp.0.ffn.experts.15.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -72508,7 +69146,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.95.w1": { + "layers.60.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -72552,7 +69190,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.329.w2": { + "layers.58.attn.wq_a": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -72596,7 +69234,7 @@ "bias": null, "target_device": null }, - "layers.40.ffn.shared_experts.w3": { + "layers.49.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -72640,43 +69278,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.191.w3": { + "mtp.0.ffn.experts.152.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -72684,43 +69319,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.228.w1": { + "mtp.0.ffn.experts.72.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -72728,43 +69360,40 @@ "bias": null, "target_device": null }, - "layers.47.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.134.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -72772,43 +69401,40 @@ "bias": null, "target_device": null }, - "layers.7.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.285.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -72816,7 +69442,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.307.w2": { + "layers.4.ffn.shared_experts.w2": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -72860,43 +69486,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.67.w3": { + "mtp.0.ffn.experts.202.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -72904,43 +69527,40 @@ "bias": null, "target_device": null }, - "layers.14.attn.indexer.wq_b": { + "mtp.0.ffn.experts.126.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -72948,43 +69568,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.318.w2": { + "mtp.0.ffn.experts.144.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -72992,43 +69609,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.320.w1": { + "mtp.0.ffn.experts.61.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -73036,7 +69650,7 @@ "bias": null, "target_device": null }, - "layers.1.ffn.shared_experts.w3": { + "layers.36.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -73080,7 +69694,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.290.w2": { + "layers.9.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -73124,7 +69738,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.80.w2": { + "layers.13.attn.wq_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -73168,43 +69782,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.373.w1": { + "mtp.0.ffn.experts.140.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -73212,43 +69823,40 @@ "bias": null, "target_device": null }, - "layers.8.ffn.shared_experts.w1": { + "mtp.0.ffn.experts.85.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -73256,43 +69864,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.261.w3": { + "mtp.0.ffn.experts.325.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -73300,43 +69905,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.303.w2": { + "mtp.0.ffn.experts.306.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -73344,7 +69946,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.358.w2": { + "layers.34.ffn.shared_experts.w3": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -73388,43 +69990,40 @@ "bias": null, "target_device": null }, - "layers.12.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.302.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -73432,7 +70031,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.332.w1": { + "layers.1.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -73476,43 +70075,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.37.w2": { + "mtp.0.ffn.experts.333.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -73520,43 +70116,40 @@ "bias": null, "target_device": null }, - "layers.25.attn.wo_a": { + "mtp.0.ffn.experts.62.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", - "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "dtype": "fp4", + "is_dynamic": false, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -73564,43 +70157,40 @@ "bias": null, "target_device": null }, - "layers.39.attn.wo_a": { + "mtp.0.ffn.experts.377.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -73608,43 +70198,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.116.w1": { + "mtp.0.ffn.experts.354.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -73652,43 +70239,40 @@ "bias": null, "target_device": null }, - "layers.50.ffn.shared_experts.w3": { + "mtp.0.ffn.experts.27.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -73696,43 +70280,40 @@ "bias": null, "target_device": null }, - "layers.14.attn.wkv": { + "mtp.0.ffn.experts.351.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -73740,7 +70321,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.284.w3": { + "layers.0.ffn.shared_experts.w1": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -73784,43 +70365,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.382.w2": { + "mtp.0.ffn.experts.254.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -73828,43 +70406,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.306.w3": { + "mtp.0.ffn.experts.234.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -73872,43 +70447,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.367.w1": { + "mtp.0.ffn.experts.83.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -73916,43 +70488,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.135.w1": { + "mtp.0.ffn.experts.297.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -73960,7 +70529,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.293.w2": { + "layers.4.attn.wo_b": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -74004,43 +70573,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.353.w1": { + "mtp.0.ffn.experts.149.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -74048,43 +70614,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.206.w3": { + "mtp.0.ffn.experts.8.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -74092,7 +70655,7 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.37.w1": { + "layers.31.attn.wkv": { "input_tensors": { "dtype": "fp8_e4m3", "is_dynamic": true, @@ -74136,43 +70699,40 @@ "bias": null, "target_device": null }, - "layers.33.ffn.shared_experts.w2": { + "mtp.0.ffn.experts.310.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -74180,43 +70740,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.273.w2": { + "mtp.0.ffn.experts.90.w1": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -74224,43 +70781,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.326.w1": { + "mtp.0.ffn.experts.284.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -74268,43 +70822,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.351.w2": { + "mtp.0.ffn.experts.168.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -74312,43 +70863,40 @@ "bias": null, "target_device": null }, - "layers.3.attn.wkv": { + "mtp.0.ffn.experts.241.w3": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -74356,43 +70904,40 @@ "bias": null, "target_device": null }, - "mtp.0.ffn.experts.295.w1": { + "mtp.0.ffn.experts.249.w2": { "input_tensors": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": true, "qscheme": "per_group", "ch_axis": -1, - "group_size": 128, + "group_size": 32, "block_size": null, - "symmetric": true, + "symmetric": null, "round_method": "half_even", - "scale_type": null, - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerGroupMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 }, "output_tensors": null, "weight": { - "dtype": "fp8_e4m3", + "dtype": "fp4", "is_dynamic": false, - "qscheme": "per_block", - "ch_axis": null, - "group_size": null, - "block_size": [ - 128, - 128 - ], - "symmetric": true, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": null, + "symmetric": null, "round_method": "half_even", - "scale_type": "float8_e8m0fnu", - "scale_format": null, - "scale_calculation_mode": null, + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": null, - "observer_cls": "PerBlock2DMinMaxObserver", + "observer_cls": "PerBlockMXObserver", "is_scale_quant": false, "enable_buffer_reuse": false, "max_input_numel": 4194304 @@ -74404,7 +70949,7 @@ "kv_cache_quant_config": {}, "kv_cache_post_rope": false, "quant_mode": "eager_mode", - "version": "0.12+0242f3b3a8c", + "version": "0.13+25771f57eb0", "export": { "kv_cache_group": [], "min_kv_scale": 0.0,