{ "op": "com.microsoft.QMoE", "cases": [ { "name": "q4_colwise_default_relu", "provenance": { "notes": "Exercises every exact default at once: 4-bit integer weights, k=1, ReLU, column-wise scales, dense routing, and raw provider layout." }, "inputs": { "inputT": { "dtype": "float32", "shape": [2, 4], "data": { "kind": "values", "values": [1.0, -2.0, 0.5, 3.0, -1.0, 2.0, 4.0, -0.5] } }, "routerT": { "dtype": "float32", "shape": [2, 3], "data": { "kind": "values", "values": [0.1, 1.2, -0.4, 2.0, -1.0, 0.5] } }, "fc1T": { "dtype": "uint8", "shape": [3, 4, 2], "data": { "kind": "cycle", "values": [154, 104, 124, 185, 39, 211, 78] } }, "fc1ScalesT": { "dtype": "float32", "shape": [3, 4], "data": { "kind": "linspace", "start": 0.05, "end": 0.3 } }, "fc2T": { "dtype": "uint8", "shape": [3, 4, 2], "data": { "kind": "cycle", "values": [78, 211, 39, 185, 124, 104, 154] } }, "fc2ScalesT": { "dtype": "float32", "shape": [3, 4], "data": { "kind": "linspace", "start": 0.04, "end": 0.22 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [2, 4], "tolerance": 0.00002 } } }, { "name": "q4_top1_tie_selects_higher_expert", "provenance": { "notes": "Pins ONNX Runtime's descending `(logit, expert_index)` ordering with selected-weight normalization: equal logits select expert 1, whose hand-computed output is [16, 32], rather than the all-zero expert 0." }, "attrs": { "normalize_routing_weights": 1 }, "inputs": { "inputT": { "dtype": "float32", "shape": [1, 2], "data": { "kind": "values", "values": [1.0, 1.0] } }, "routerT": { "dtype": "float32", "shape": [1, 2], "data": { "kind": "values", "values": [0.0, 0.0] } }, "fc1T": { "dtype": "uint8", "shape": [2, 2, 1], "data": { "kind": "values", "values": [136, 136, 170, 170] } }, "fc1ScalesT": { "dtype": "float32", "shape": [2, 2], "data": { "kind": "constant", "value": 1.0 } }, "fc2T": { "dtype": "uint8", "shape": [2, 2, 1], "data": { "kind": "values", "values": [136, 136, 170, 204] } }, "fc2ScalesT": { "dtype": "float32", "shape": [2, 2], "data": { "kind": "constant", "value": 1.0 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [1, 2], "data": { "kind": "values", "values": [16.0, 32.0] }, "tolerance": 0 } } }, { "name": "q8_colwise_swiglu", "provenance": { "notes": "Covers 8-bit integer weights and the supported interleaved SwiGLU mode with non-default alpha and beta." }, "attrs": { "activation_alpha": 1.1, "activation_beta": 0.05, "activation_type": "swiglu", "expert_weight_bits": 8, "swiglu_fusion": 1 }, "inputs": { "inputT": { "dtype": "float32", "shape": [3, 4], "data": { "kind": "fillFloat32", "sinStep": 0.17, "cosStep": 0.31, "scale": 0.3 } }, "routerT": { "dtype": "float32", "shape": [3, 3], "data": { "kind": "fillFloat32", "sinStep": 0.37, "cosStep": 0.19, "scale": 0.7 } }, "fc1T": { "dtype": "uint8", "shape": [3, 6, 4], "data": { "kind": "cycle", "values": [131, 96, 174, 118, 205, 83, 149] } }, "fc1ScalesT": { "dtype": "float32", "shape": [3, 6], "data": { "kind": "linspace", "start": 0.006, "end": 0.042 } }, "fc2T": { "dtype": "uint8", "shape": [3, 4, 3], "data": { "kind": "cycle", "values": [149, 83, 205, 118, 174, 96, 131] } }, "fc2ScalesT": { "dtype": "float32", "shape": [3, 4], "data": { "kind": "linspace", "start": 0.008, "end": 0.037 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [3, 4], "tolerance": 0.00002 } } }, { "name": "q4_blockwise_topk2_swiglu", "provenance": { "notes": "Two complete 16-element quantization blocks on both reductions, with two selected experts and stable top-k softmax mixing." }, "attrs": { "activation_type": "swiglu", "block_size": 16, "k": 2, "normalize_routing_weights": 1, "swiglu_fusion": 1, "weights_prepacked": 0 }, "inputs": { "inputT": { "dtype": "float32", "shape": [4, 32], "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.37, "scale": 0.4 } }, "routerT": { "dtype": "float32", "shape": [4, 3], "data": { "kind": "fillFloat32", "sinStep": 0.43, "cosStep": 0.23, "scale": 0.6 } }, "fc1T": { "dtype": "uint8", "shape": [3, 64, 16], "data": { "kind": "cycle", "values": [154, 104, 124, 185, 39, 211, 78] } }, "fc1ScalesT": { "dtype": "float32", "shape": [3, 64, 2], "data": { "kind": "linspace", "start": 0.04, "end": 0.33 } }, "fc2T": { "dtype": "uint8", "shape": [3, 32, 16], "data": { "kind": "cycle", "values": [78, 211, 39, 185, 124, 104, 154] } }, "fc2ScalesT": { "dtype": "float32", "shape": [3, 32, 2], "data": { "kind": "linspace", "start": 0.06, "end": 0.27 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [4, 32], "tolerance": 0.00005 } } }, { "name": "q4_rank3_input", "provenance": { "notes": "Locks the standard rank-3 input form and flattening of batch and sequence into router rows." }, "inputs": { "inputT": { "dtype": "float32", "shape": [2, 2, 4], "data": { "kind": "fillFloat32", "sinStep": 0.21, "cosStep": 0.13, "scale": 0.4 } }, "routerT": { "dtype": "float32", "shape": [4, 2], "data": { "kind": "fillFloat32", "sinStep": 0.41, "cosStep": 0.17, "scale": 0.8 } }, "fc1T": { "dtype": "uint8", "shape": [2, 4, 2], "data": { "kind": "cycle", "values": [25, 118, 203, 71, 154] } }, "fc1ScalesT": { "dtype": "float32", "shape": [2, 4], "data": { "kind": "linspace", "start": 0.04, "end": 0.2 } }, "fc2T": { "dtype": "uint8", "shape": [2, 4, 2], "data": { "kind": "cycle", "values": [97, 44, 218, 131, 76] } }, "fc2ScalesT": { "dtype": "float32", "shape": [2, 4], "data": { "kind": "linspace", "start": 0.03, "end": 0.18 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [2, 2, 4], "tolerance": 0.00002 } } }, { "name": "q8_swiglu_limit", "provenance": { "notes": "Exercises the optional finite SwiGLU clamp and the explicit 8-bit path." }, "attrs": { "activation_alpha": 0.9, "activation_beta": -0.02, "activation_type": "swiglu", "expert_weight_bits": 8, "swiglu_fusion": 1, "swiglu_limit": 0.75 }, "inputs": { "inputT": { "dtype": "float32", "shape": [2, 4], "data": { "kind": "values", "values": [2.0, -3.0, 1.0, 4.0, -1.0, 0.5, 3.0, -2.0] } }, "routerT": { "dtype": "float32", "shape": [2, 2], "data": { "kind": "values", "values": [1.0, 0.0, -1.0, 2.0] } }, "fc1T": { "dtype": "uint8", "shape": [2, 8, 4], "data": { "kind": "cycle", "values": [131, 96, 174, 118, 205, 83, 149] } }, "fc1ScalesT": { "dtype": "float32", "shape": [2, 8], "data": { "kind": "linspace", "start": 0.02, "end": 0.12 } }, "fc2T": { "dtype": "uint8", "shape": [2, 4, 4], "data": { "kind": "cycle", "values": [149, 83, 205, 118, 174, 96, 131] } }, "fc2ScalesT": { "dtype": "float32", "shape": [2, 4], "data": { "kind": "linspace", "start": 0.03, "end": 0.11 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [2, 4], "tolerance": 0.00002 } } }, { "name": "q4_zero_inter", "provenance": { "notes": "A zero intermediate dimension has an exact zero result without binding empty weight buffers in a shader." }, "attrs": { "activation_type": "swiglu", "swiglu_fusion": 1 }, "inputs": { "inputT": { "dtype": "float32", "shape": [3, 4], "data": { "kind": "fillFloat32", "sinStep": 0.17, "cosStep": 0.31, "scale": 0.3 } }, "routerT": { "dtype": "float32", "shape": [3, 3], "data": { "kind": "fillFloat32", "sinStep": 0.37, "cosStep": 0.19, "scale": 0.7 } }, "fc1T": { "dtype": "uint8", "shape": [3, 0, 2], "data": { "kind": "values", "values": [] } }, "fc1ScalesT": { "dtype": "float32", "shape": [3, 0], "data": { "kind": "values", "values": [] } }, "fc2T": { "dtype": "uint8", "shape": [3, 4, 0], "data": { "kind": "values", "values": [] } }, "fc2ScalesT": { "dtype": "float32", "shape": [3, 4], "data": { "kind": "linspace", "start": 0.008, "end": 0.037 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [3, 4], "data": { "kind": "constant", "value": 0.0 }, "tolerance": 0 } } }, { "name": "empty_zero_dim", "provenance": { "notes": "Zero tokens in the rank-2 form must retain the routed quantized shader identities while every output dispatch is elided." }, "inputs": { "inputT": { "dtype": "float32", "shape": [0, 4], "data": { "kind": "values", "values": [] } }, "routerT": { "dtype": "float32", "shape": [0, 3], "data": { "kind": "values", "values": [] } }, "fc1T": { "dtype": "uint8", "shape": [3, 4, 2], "data": { "kind": "constant", "value": 120 } }, "fc1ScalesT": { "dtype": "float32", "shape": [3, 4], "data": { "kind": "constant", "value": 0.05 } }, "fc2T": { "dtype": "uint8", "shape": [3, 4, 2], "data": { "kind": "constant", "value": 130 } }, "fc2ScalesT": { "dtype": "float32", "shape": [3, 4], "data": { "kind": "constant", "value": 0.05 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [0, 4], "data": { "kind": "values", "values": [] }, "tolerance": 0 } } }, { "name": "q4_colwise_relu_deep_reduction", "provenance": { "notes": "A 128-long reduction on both projections, deep enough that splitting it across lanes is worthwhile; the shallow cases above keep the one-thread-per-column schedule." }, "attrs": { "weights_prepacked": 0 }, "inputs": { "inputT": { "dtype": "float32", "shape": [2, 128], "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.29, "scale": 0.4 } }, "routerT": { "dtype": "float32", "shape": [2, 2], "data": { "kind": "fillFloat32", "sinStep": 0.41, "cosStep": 0.17, "scale": 0.6 } }, "fc1T": { "dtype": "uint8", "shape": [2, 128, 64], "data": { "kind": "cycle", "values": [154, 104, 124, 185, 39, 211, 78] } }, "fc1ScalesT": { "dtype": "float32", "shape": [2, 128], "data": { "kind": "linspace", "start": 0.02, "end": 0.19 } }, "fc2T": { "dtype": "uint8", "shape": [2, 128, 64], "data": { "kind": "cycle", "values": [78, 211, 39, 185, 124, 104, 154] } }, "fc2ScalesT": { "dtype": "float32", "shape": [2, 128], "data": { "kind": "linspace", "start": 0.03, "end": 0.21 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [2, 128], "tolerance": 0.00005 } } }, { "name": "q4_blockwise_swiglu_topk2_deep_reduction", "provenance": { "notes": "Interleaved SwiGLU with two selected experts over a 128-long reduction and four 32-wide quantization blocks per row." }, "attrs": { "activation_type": "swiglu", "block_size": 32, "k": 2, "normalize_routing_weights": 1, "swiglu_fusion": 1, "weights_prepacked": 0 }, "inputs": { "inputT": { "dtype": "float32", "shape": [3, 128], "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.37, "scale": 0.4 } }, "routerT": { "dtype": "float32", "shape": [3, 3], "data": { "kind": "fillFloat32", "sinStep": 0.43, "cosStep": 0.23, "scale": 0.6 } }, "fc1T": { "dtype": "uint8", "shape": [3, 256, 64], "data": { "kind": "cycle", "values": [97, 182, 45, 213, 131, 68, 159, 22, 241] } }, "fc1ScalesT": { "dtype": "float32", "shape": [3, 256, 4], "data": { "kind": "linspace", "start": 0.04, "end": 0.33 } }, "fc2T": { "dtype": "uint8", "shape": [3, 128, 64], "data": { "kind": "cycle", "values": [241, 22, 159, 68, 131, 213, 45, 182, 97] } }, "fc2ScalesT": { "dtype": "float32", "shape": [3, 128, 4], "data": { "kind": "linspace", "start": 0.06, "end": 0.27 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [3, 128], "tolerance": 0.00005 } } }, { "name": "q8_swiglu_deep_reduction", "provenance": { "notes": "Eight-bit codes over the same 128-long reduction: one stored word per logical column instead of a shared nibble pair." }, "attrs": { "activation_type": "swiglu", "expert_weight_bits": 8, "swiglu_fusion": 1, "weights_prepacked": 0 }, "inputs": { "inputT": { "dtype": "float32", "shape": [2, 128], "data": { "kind": "fillFloat32", "sinStep": 0.23, "cosStep": 0.11, "scale": 0.35 } }, "routerT": { "dtype": "float32", "shape": [2, 2], "data": { "kind": "fillFloat32", "sinStep": 0.31, "cosStep": 0.47, "scale": 0.6 } }, "fc1T": { "dtype": "uint8", "shape": [2, 256, 128], "data": { "kind": "cycle", "values": [113, 26, 201, 88, 167, 54, 229, 141, 73] } }, "fc1ScalesT": { "dtype": "float32", "shape": [2, 256], "data": { "kind": "linspace", "start": 0.01, "end": 0.09 } }, "fc2T": { "dtype": "uint8", "shape": [2, 128, 128], "data": { "kind": "cycle", "values": [73, 141, 229, 54, 167, 88, 201, 26, 113] } }, "fc2ScalesT": { "dtype": "float32", "shape": [2, 128], "data": { "kind": "linspace", "start": 0.02, "end": 0.08 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [2, 128], "tolerance": 0.00005 } } }, { "name": "q8_swiglu_decode_gemv_odd_reduction", "provenance": { "notes": "An 8-bit decode on the cooperative GEMV schedule with an ODD hidden size. The reduction steps two columns at a time -- one stored word per column at 8 bits, one word for both at 4 -- so an odd length leaves a final column that only 8-bit weights can produce." }, "attrs": { "activation_type": "swiglu", "expert_weight_bits": 8, "swiglu_fusion": 1, "weights_prepacked": 0 }, "inputs": { "inputT": { "dtype": "float32", "shape": [1, 129], "data": { "kind": "fillFloat32", "sinStep": 0.261, "cosStep": 0.11, "scale": 0.35 } }, "routerT": { "dtype": "float32", "shape": [1, 2], "data": { "kind": "fillFloat32", "sinStep": 0.341, "cosStep": 0.47, "scale": 0.6 } }, "fc1T": { "dtype": "uint8", "shape": [2, 256, 129], "data": { "kind": "cycle", "values": [113, 26, 201, 88, 167, 54, 229, 141, 73] } }, "fc1ScalesT": { "dtype": "float32", "shape": [2, 256], "data": { "kind": "linspace", "start": 0.01, "end": 0.09 } }, "fc2T": { "dtype": "uint8", "shape": [2, 129, 128], "data": { "kind": "cycle", "values": [73, 141, 229, 54, 167, 88, 201, 26, 113] } }, "fc2ScalesT": { "dtype": "float32", "shape": [2, 129], "data": { "kind": "linspace", "start": 0.02, "end": 0.08 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [1, 129], "tolerance": 0.00005 } } }, { "name": "q4_grouped_prefill_relu", "provenance": { "notes": "Enough routed slots per expert to fill grouped tiles, so one dequantized weight tile is reused across a tile of slots." }, "attrs": { "weights_prepacked": 0 }, "inputs": { "inputT": { "dtype": "float32", "shape": [96, 64], "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.29, "scale": 0.4 } }, "routerT": { "dtype": "float32", "shape": [96, 3], "data": { "kind": "fillFloat32", "sinStep": 0.33999999999999997, "cosStep": 0.24, "scale": 0.6 } }, "fc1T": { "dtype": "uint8", "shape": [3, 64, 32], "data": { "kind": "cycle", "values": [97, 182, 45, 213, 131, 68, 159, 22, 241] } }, "fc1ScalesT": { "dtype": "float32", "shape": [3, 64], "data": { "kind": "linspace", "start": 0.02, "end": 0.21 } }, "fc2T": { "dtype": "uint8", "shape": [3, 64, 32], "data": { "kind": "cycle", "values": [241, 22, 159, 68, 131, 213, 45, 182, 97] } }, "fc2ScalesT": { "dtype": "float32", "shape": [3, 64], "data": { "kind": "linspace", "start": 0.03, "end": 0.18 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [96, 64], "tolerance": 0.00005 } } }, { "name": "q4_grouped_prefill_swiglu_topk2", "provenance": { "notes": "Enough routed slots per expert to fill grouped tiles, so one dequantized weight tile is reused across a tile of slots." }, "attrs": { "activation_type": "swiglu", "k": 2, "normalize_routing_weights": 1, "swiglu_fusion": 1, "weights_prepacked": 0 }, "inputs": { "inputT": { "dtype": "float32", "shape": [48, 64], "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.37, "scale": 0.4 } }, "routerT": { "dtype": "float32", "shape": [48, 3], "data": { "kind": "fillFloat32", "sinStep": 0.4, "cosStep": 0.32, "scale": 0.6 } }, "fc1T": { "dtype": "uint8", "shape": [3, 128, 32], "data": { "kind": "cycle", "values": [97, 182, 45, 213, 131, 68, 159, 22, 241] } }, "fc1ScalesT": { "dtype": "float32", "shape": [3, 128], "data": { "kind": "linspace", "start": 0.02, "end": 0.21 } }, "fc2T": { "dtype": "uint8", "shape": [3, 64, 32], "data": { "kind": "cycle", "values": [241, 22, 159, 68, 131, 213, 45, 182, 97] } }, "fc2ScalesT": { "dtype": "float32", "shape": [3, 64], "data": { "kind": "linspace", "start": 0.03, "end": 0.18 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [48, 64], "tolerance": 0.00005 } } }, { "name": "q4_grouped_prefill_partial_tiles", "provenance": { "notes": "Half the routed slots the grouped schedule used to demand: 3 experts share 64 slots against 32-row tiles, so most of every tile is padding and the store guard drops more rows than it keeps." }, "attrs": { "activation_type": "swiglu", "k": 2, "normalize_routing_weights": 1, "swiglu_fusion": 1, "weights_prepacked": 0 }, "inputs": { "inputT": { "dtype": "float32", "shape": [32, 64], "data": { "kind": "fillFloat32", "sinStep": 0.23, "cosStep": 0.41, "scale": 0.4 } }, "routerT": { "dtype": "float32", "shape": [32, 3], "data": { "kind": "fillFloat32", "sinStep": 0.34, "cosStep": 0.28, "scale": 0.6 } }, "fc1T": { "dtype": "uint8", "shape": [3, 128, 32], "data": { "kind": "cycle", "values": [97, 182, 45, 213, 131, 68, 159, 22, 241] } }, "fc1ScalesT": { "dtype": "float32", "shape": [3, 128], "data": { "kind": "linspace", "start": 0.02, "end": 0.21 } }, "fc2T": { "dtype": "uint8", "shape": [3, 64, 32], "data": { "kind": "cycle", "values": [241, 22, 159, 68, 131, 213, 45, 182, 97] } }, "fc2ScalesT": { "dtype": "float32", "shape": [3, 64], "data": { "kind": "linspace", "start": 0.03, "end": 0.18 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [32, 64], "tolerance": 0.00005 } } }, { "name": "q8_grouped_prefill_swiglu_blockwise", "provenance": { "notes": "Enough routed slots per expert to fill grouped tiles, so one dequantized weight tile is reused across a tile of slots. The tiled reduction sums in a different order than the reference, so the tolerance carries that f32 regrouping; outputs are order 0.4, which a 10% error would move by 0.04." }, "attrs": { "activation_type": "swiglu", "block_size": 32, "k": 2, "normalize_routing_weights": 1, "swiglu_fusion": 1, "weights_prepacked": 0, "expert_weight_bits": 8 }, "inputs": { "inputT": { "dtype": "float32", "shape": [48, 64], "data": { "kind": "fillFloat32", "sinStep": 0.23, "cosStep": 0.17, "scale": 0.4 } }, "routerT": { "dtype": "float32", "shape": [48, 3], "data": { "kind": "fillFloat32", "sinStep": 0.44, "cosStep": 0.12000000000000001, "scale": 0.6 } }, "fc1T": { "dtype": "uint8", "shape": [3, 128, 64], "data": { "kind": "cycle", "values": [97, 182, 45, 213, 131, 68, 159, 22, 241] } }, "fc1ScalesT": { "dtype": "float32", "shape": [3, 128, 2], "data": { "kind": "linspace", "start": 0.02, "end": 0.21 } }, "fc2T": { "dtype": "uint8", "shape": [3, 64, 64], "data": { "kind": "cycle", "values": [241, 22, 159, 68, 131, 213, 45, 182, 97] } }, "fc2ScalesT": { "dtype": "float32", "shape": [3, 64, 2], "data": { "kind": "linspace", "start": 0.03, "end": 0.18 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [48, 64], "tolerance": 0.0003 } } } ] }