W8A8 expert kernel (FP8 x FP8 tensor cores, K-major weights at load): 11.4 ms vs 17.7 bf16 per call; A/B results
ab8889c verified | { | |
| "base_model": "sand-ai/MAGI-2-preview", | |
| "base_revision": "2dea51b64db47ee5b4402d36fd90829a0c58913b", | |
| "component": "preview", | |
| "quant_method": "fp8", | |
| "dtype": "float8_e4m3fn", | |
| "scheme": "weight-only; one fp32 scale per (flattened head-expert row, output channel) = amax over the reduction dim / 448; round to nearest", | |
| "quantized_tensors": ["*.moe_mlp.W_gate", "*.moe_mlp.W_up", "*.moe_mlp.W_down"], | |
| "scale_key_suffix": "_scale", | |
| "scale_dtype": "float32", | |
| "scale_shapes": {"W_gate": "(rows, d_expert)", "W_up": "(rows, d_expert)", "W_down": "(rows, d_head)"}, | |
| "unchanged": "every other tensor, bf16 or fp32 as shipped", | |
| "compute": "MAGI2_FP8_MOE=w8a8 (default): expert inputs quantized to FP8 per (token, head), SwiGLU7 output at a fixed scale of 8, FP8 x FP8 -> fp32 tensor-core GEMMs on K-major weights; w8a16: FP8 tiles widened to bf16, bf16 x bf16 -> fp32", | |
| "code": "magi2_fp8/ (quantize.py, fp8_moe.py, patch.py, entry.py, check_kernel.py)" | |
| } | |