{ "op": "com.microsoft.MoE", "tunableSpace": { "workgroupSize": [64, 128], "decodeLanes": [32], "groupThreads": [8, 16], "groupRegM": [1, 2, 4], "groupRegN": [1, 2, 4], "groupTileK": [8, 16, 32] }, "cases": [ { "name": "moe-f32-t64-h256-e8-k2-i256-swiglu", "attrs": { "k": 2, "activation_type": "swiglu", "swiglu_fusion": 2, "normalize_routing_weights": 1, "activation_alpha": 1.702 }, "inputs": { "inputT": { "dtype": "float32", "shape": [64, 256], "data": { "kind": "fillFloat32", "sinStep": 0.17, "cosStep": 0.31, "scale": 0.5 } }, "routerT": { "dtype": "float32", "shape": [64, 8], "data": { "kind": "fillFloat32", "sinStep": 0.37, "cosStep": 0.19, "scale": 0.7 } }, "fc1T": { "dtype": "float32", "shape": [8, 512, 256], "data": { "kind": "fillFloat32", "sinStep": 0.11, "cosStep": 0.23, "scale": 0.05 } }, "fc2T": { "dtype": "float32", "shape": [8, 256, 256], "data": { "kind": "fillFloat32", "sinStep": 0.07, "cosStep": 0.41, "scale": 0.05 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [64, 256] } } }, { "name": "moe-f32-swiglu-t1-h1024-e8-k2-i512-decode", "preset": "smoke", "vars": { "tokens": 1, "hidden": 1024, "experts": 8, "inter": 512, "topK": 2 }, "attrs": { "k": 2, "activation_type": "swiglu", "swiglu_fusion": 2, "normalize_routing_weights": 1 }, "inputs": { "inputT": { "dtype": "float32", "shape": [1, 1024], "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.31, "scale": 0.5 } }, "routerT": { "dtype": "float32", "shape": [1, 8], "data": { "kind": "fillFloat32", "sinStep": 0.33, "cosStep": 0.19, "scale": 0.7 } }, "fc1T": { "dtype": "float32", "shape": [8, 1024, 1024], "data": { "kind": "fillFloat32", "sinStep": 0.17, "cosStep": 0.23, "scale": 0.05 } }, "fc2T": { "dtype": "float32", "shape": [8, 1024, 512], "data": { "kind": "fillFloat32", "sinStep": 0.15, "cosStep": 0.41, "scale": 0.05 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [1, 1024] } }, "bench": { "metrics": [{ "type": "bandwidth", "value": "args.topK * 3 * args.hidden * args.inter * 4" }] }, "provenance": { "notes": "Scored as bandwidth, not gflops: a decode step reads whole expert weight matrices to produce one output row, so memory is its roofline and the compute peak is the wrong denominator. The prefill cases stay on gflops." } }, { "name": "moe-f32-relu-t1-h1024-e8-k1-i1024-decode-relu", "preset": "stress", "vars": { "tokens": 1, "hidden": 1024, "experts": 8, "inter": 1024, "topK": 1 }, "attrs": { "k": 1, "activation_type": "relu", "normalize_routing_weights": 1 }, "inputs": { "inputT": { "dtype": "float32", "shape": [1, 1024], "data": { "kind": "fillFloat32", "sinStep": 0.17, "cosStep": 0.31, "scale": 0.5 } }, "routerT": { "dtype": "float32", "shape": [1, 8], "data": { "kind": "fillFloat32", "sinStep": 0.37, "cosStep": 0.19, "scale": 0.7 } }, "fc1T": { "dtype": "float32", "shape": [8, 1024, 1024], "data": { "kind": "fillFloat32", "sinStep": 0.21, "cosStep": 0.23, "scale": 0.05 } }, "fc2T": { "dtype": "float32", "shape": [8, 1024, 1024], "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.41, "scale": 0.05 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [1, 1024] } }, "bench": { "metrics": [{ "type": "bandwidth", "value": "args.topK * 2 * args.hidden * args.inter * 4" }] }, "provenance": { "notes": "Scored as bandwidth, not gflops: a decode step reads whole expert weight matrices to produce one output row, so memory is its roofline and the compute peak is the wrong denominator. The prefill cases stay on gflops." } }, { "name": "moe-f32-swiglu-t8-h1024-e8-k2-i512-decode-batch8", "preset": "stress", "vars": { "tokens": 8, "hidden": 1024, "experts": 8, "inter": 512, "topK": 2 }, "attrs": { "k": 2, "activation_type": "swiglu", "swiglu_fusion": 2, "normalize_routing_weights": 1 }, "inputs": { "inputT": { "dtype": "float32", "shape": [8, 1024], "data": { "kind": "fillFloat32", "sinStep": 0.21, "cosStep": 0.31, "scale": 0.5 } }, "routerT": { "dtype": "float32", "shape": [8, 8], "data": { "kind": "fillFloat32", "sinStep": 0.41, "cosStep": 0.19, "scale": 0.7 } }, "fc1T": { "dtype": "float32", "shape": [8, 1024, 1024], "data": { "kind": "fillFloat32", "sinStep": 0.25, "cosStep": 0.23, "scale": 0.05 } }, "fc2T": { "dtype": "float32", "shape": [8, 1024, 512], "data": { "kind": "fillFloat32", "sinStep": 0.23, "cosStep": 0.41, "scale": 0.05 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [8, 1024] } }, "bench": { "metrics": [{ "type": "bandwidth", "value": "args.topK * 3 * args.hidden * args.inter * 4" }] }, "provenance": { "notes": "Scored as bandwidth, not gflops: a decode step reads whole expert weight matrices to produce one output row, so memory is its roofline and the compute peak is the wrong denominator. The prefill cases stay on gflops." } }, { "name": "moe-f32-swiglu-t256-h1024-e8-k2-i512-prefill", "preset": "stress", "vars": { "tokens": 256, "hidden": 1024, "experts": 8, "inter": 512, "topK": 2 }, "attrs": { "k": 2, "activation_type": "swiglu", "swiglu_fusion": 2, "normalize_routing_weights": 1 }, "inputs": { "inputT": { "dtype": "float32", "shape": [256, 1024], "data": { "kind": "fillFloat32", "sinStep": 0.29, "cosStep": 0.31, "scale": 0.5 } }, "routerT": { "dtype": "float32", "shape": [256, 8], "data": { "kind": "fillFloat32", "sinStep": 0.49, "cosStep": 0.19, "scale": 0.7 } }, "fc1T": { "dtype": "float32", "shape": [8, 1024, 1024], "data": { "kind": "fillFloat32", "sinStep": 0.33, "cosStep": 0.23, "scale": 0.05 } }, "fc2T": { "dtype": "float32", "shape": [8, 1024, 512], "data": { "kind": "fillFloat32", "sinStep": 0.31, "cosStep": 0.41, "scale": 0.05 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [256, 1024] } }, "bench": { "metrics": [{ "type": "gflops", "value": "2 * args.tokens * args.topK * args.hidden * args.inter * 3" }] } }, { "name": "moe-f32-swiglu-t32-h1024-e8-k2-i512-prefill-small", "preset": "stress", "vars": { "tokens": 32, "hidden": 1024, "experts": 8, "inter": 512, "topK": 2 }, "attrs": { "k": 2, "activation_type": "swiglu", "swiglu_fusion": 2, "normalize_routing_weights": 1 }, "inputs": { "inputT": { "dtype": "float32", "shape": [32, 1024], "data": { "kind": "fillFloat32", "sinStep": 0.23, "cosStep": 0.31, "scale": 0.5 } }, "routerT": { "dtype": "float32", "shape": [32, 8], "data": { "kind": "fillFloat32", "sinStep": 0.49, "cosStep": 0.19, "scale": 0.7 } }, "fc1T": { "dtype": "float32", "shape": [8, 1024, 1024], "data": { "kind": "fillFloat32", "sinStep": 0.33, "cosStep": 0.23, "scale": 0.05 } }, "fc2T": { "dtype": "float32", "shape": [8, 1024, 512], "data": { "kind": "fillFloat32", "sinStep": 0.31, "cosStep": 0.41, "scale": 0.05 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [32, 1024] } }, "bench": { "metrics": [{ "type": "gflops", "value": "2 * args.tokens * args.topK * args.hidden * args.inter * 3" }] } }, { "name": "moe-f32-swiglu-t64-h1024-e8-k2-i512-prefill-small", "preset": "stress", "vars": { "tokens": 64, "hidden": 1024, "experts": 8, "inter": 512, "topK": 2 }, "attrs": { "k": 2, "activation_type": "swiglu", "swiglu_fusion": 2, "normalize_routing_weights": 1 }, "inputs": { "inputT": { "dtype": "float32", "shape": [64, 1024], "data": { "kind": "fillFloat32", "sinStep": 0.27, "cosStep": 0.31, "scale": 0.5 } }, "routerT": { "dtype": "float32", "shape": [64, 8], "data": { "kind": "fillFloat32", "sinStep": 0.49, "cosStep": 0.19, "scale": 0.7 } }, "fc1T": { "dtype": "float32", "shape": [8, 1024, 1024], "data": { "kind": "fillFloat32", "sinStep": 0.33, "cosStep": 0.23, "scale": 0.05 } }, "fc2T": { "dtype": "float32", "shape": [8, 1024, 512], "data": { "kind": "fillFloat32", "sinStep": 0.31, "cosStep": 0.41, "scale": 0.05 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [64, 1024] } }, "bench": { "metrics": [{ "type": "gflops", "value": "2 * args.tokens * args.topK * args.hidden * args.inter * 3" }] } }, { "name": "moe-f32-swiglu-t96-h1024-e8-k2-i512-prefill-small", "preset": "stress", "vars": { "tokens": 96, "hidden": 1024, "experts": 8, "inter": 512, "topK": 2 }, "attrs": { "k": 2, "activation_type": "swiglu", "swiglu_fusion": 2, "normalize_routing_weights": 1 }, "inputs": { "inputT": { "dtype": "float32", "shape": [96, 1024], "data": { "kind": "fillFloat32", "sinStep": 0.21, "cosStep": 0.31, "scale": 0.5 } }, "routerT": { "dtype": "float32", "shape": [96, 8], "data": { "kind": "fillFloat32", "sinStep": 0.49, "cosStep": 0.19, "scale": 0.7 } }, "fc1T": { "dtype": "float32", "shape": [8, 1024, 1024], "data": { "kind": "fillFloat32", "sinStep": 0.33, "cosStep": 0.23, "scale": 0.05 } }, "fc2T": { "dtype": "float32", "shape": [8, 1024, 512], "data": { "kind": "fillFloat32", "sinStep": 0.31, "cosStep": 0.41, "scale": 0.05 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [96, 1024] } }, "bench": { "metrics": [{ "type": "gflops", "value": "2 * args.tokens * args.topK * args.hidden * args.inter * 3" }] } }, { "name": "moe-f32-swiglu-t128-h1024-e8-k2-i512-prefill-small", "preset": "stress", "vars": { "tokens": 128, "hidden": 1024, "experts": 8, "inter": 512, "topK": 2 }, "attrs": { "k": 2, "activation_type": "swiglu", "swiglu_fusion": 2, "normalize_routing_weights": 1 }, "inputs": { "inputT": { "dtype": "float32", "shape": [128, 1024], "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.31, "scale": 0.5 } }, "routerT": { "dtype": "float32", "shape": [128, 8], "data": { "kind": "fillFloat32", "sinStep": 0.49, "cosStep": 0.19, "scale": 0.7 } }, "fc1T": { "dtype": "float32", "shape": [8, 1024, 1024], "data": { "kind": "fillFloat32", "sinStep": 0.33, "cosStep": 0.23, "scale": 0.05 } }, "fc2T": { "dtype": "float32", "shape": [8, 1024, 512], "data": { "kind": "fillFloat32", "sinStep": 0.31, "cosStep": 0.41, "scale": 0.05 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [128, 1024] } }, "bench": { "metrics": [{ "type": "gflops", "value": "2 * args.tokens * args.topK * args.hidden * args.inter * 3" }] } }, { "name": "moe-f32-swiglu-t48-h1024-e8-k2-i1024-prefill-band", "preset": "stress", "vars": { "tokens": 48, "hidden": 1024, "experts": 8, "inter": 1024, "topK": 2 }, "attrs": { "k": 2, "activation_type": "swiglu", "swiglu_fusion": 2, "normalize_routing_weights": 1 }, "inputs": { "inputT": { "dtype": "float32", "shape": [48, 1024], "data": { "kind": "fillFloat32", "sinStep": 0.37, "cosStep": 0.31, "scale": 0.5 } }, "routerT": { "dtype": "float32", "shape": [48, 8], "data": { "kind": "fillFloat32", "sinStep": 0.47, "cosStep": 0.19, "scale": 0.7 } }, "fc1T": { "dtype": "float32", "shape": [8, 2048, 1024], "data": { "kind": "fillFloat32", "sinStep": 0.33, "cosStep": 0.23, "scale": 0.05 } }, "fc2T": { "dtype": "float32", "shape": [8, 1024, 1024], "data": { "kind": "fillFloat32", "sinStep": 0.31, "cosStep": 0.41, "scale": 0.05 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [48, 1024] } }, "bench": { "metrics": [{ "type": "gflops", "value": "2 * args.tokens * args.topK * args.hidden * args.inter * 3" }] } }, { "name": "moe-f32-swiglu-t96-h1024-e16-k2-i512-prefill-band", "preset": "stress", "vars": { "tokens": 96, "hidden": 1024, "experts": 16, "inter": 512, "topK": 2 }, "attrs": { "k": 2, "activation_type": "swiglu", "swiglu_fusion": 2, "normalize_routing_weights": 1 }, "inputs": { "inputT": { "dtype": "float32", "shape": [96, 1024], "data": { "kind": "fillFloat32", "sinStep": 0.43, "cosStep": 0.31, "scale": 0.5 } }, "routerT": { "dtype": "float32", "shape": [96, 16], "data": { "kind": "fillFloat32", "sinStep": 0.53, "cosStep": 0.19, "scale": 0.7 } }, "fc1T": { "dtype": "float32", "shape": [16, 1024, 1024], "data": { "kind": "fillFloat32", "sinStep": 0.33, "cosStep": 0.23, "scale": 0.05 } }, "fc2T": { "dtype": "float32", "shape": [16, 1024, 512], "data": { "kind": "fillFloat32", "sinStep": 0.31, "cosStep": 0.41, "scale": 0.05 } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [96, 1024] } }, "bench": { "metrics": [{ "type": "gflops", "value": "2 * args.tokens * args.topK * args.hidden * args.inter * 3" }] } }, { "name": "moe-f32-qwen3-moe-decode-t1-h2048-e32-k8-i768", "preset": "model", "provenance": { "notes": "Qwen3-MoE class defaults (hidden_size 2048, moe_intermediate_size 768, num_experts_per_tok 8); expert count capped at 32 of the config's 128 to keep each weight tensor under 512 MB. Scored as bandwidth, not gflops: a decode step reads whole expert weight matrices to produce one output row, so memory is its roofline and the compute peak is the wrong denominator. The prefill cases stay on gflops." }, "vars": { "tokens": 1, "hidden": 2048, "experts": 32, "inter": 768, "topK": 8 }, "attrs": { "k": 8, "activation_type": "swiglu", "swiglu_fusion": 2, "normalize_routing_weights": 1 }, "inputs": { "inputT": { "shape": [1, 2048], "dtype": "float32", "dist": "normal", "seed": 3011, "scale": 0.5 }, "routerT": { "shape": [1, 32], "dtype": "float32", "dist": "normal", "seed": 3012, "scale": 0.7 }, "fc1T": { "shape": [32, 1536, 2048], "dtype": "float32", "dist": "normal", "seed": 3013, "scale": 0.05 }, "fc2T": { "shape": [32, 2048, 768], "dtype": "float32", "dist": "normal", "seed": 3014, "scale": 0.05 } }, "outputs": { "outputT": { "shape": [1, 2048], "dtype": "float32" } }, "bench": { "metrics": [{ "type": "bandwidth", "value": "args.topK * 3 * args.hidden * args.inter * 4" }] } }, { "name": "moe-f32-qwen3-moe-prefill-t256-h2048-e32-k8-i768", "preset": "model", "provenance": { "notes": "Qwen3-MoE class defaults at a 256-token prefill chunk; 2048 routed slots put every expert over the grouped schedule's slot floor." }, "vars": { "tokens": 256, "hidden": 2048, "experts": 32, "inter": 768, "topK": 8 }, "attrs": { "k": 8, "activation_type": "swiglu", "swiglu_fusion": 2, "normalize_routing_weights": 1 }, "inputs": { "inputT": { "shape": [256, 2048], "dtype": "float32", "dist": "normal", "seed": 3021, "scale": 0.5 }, "routerT": { "shape": [256, 32], "dtype": "float32", "dist": "normal", "seed": 3022, "scale": 0.7 }, "fc1T": { "shape": [32, 1536, 2048], "dtype": "float32", "dist": "normal", "seed": 3023, "scale": 0.05 }, "fc2T": { "shape": [32, 2048, 768], "dtype": "float32", "dist": "normal", "seed": 3024, "scale": 0.05 } }, "outputs": { "outputT": { "shape": [256, 2048], "dtype": "float32" } }, "bench": { "metrics": [{ "type": "gflops", "value": "2 * args.tokens * args.topK * args.hidden * args.inter * 3" }] } }, { "name": "moe-f32-olmoe-decode-t1-h2048-e16-k8-i2048", "preset": "model", "provenance": { "notes": "OLMoE class defaults (hidden_size 2048, intermediate_size 2048, num_experts_per_tok 8); expert count capped at 16 of the config's 64. Scored as bandwidth, not gflops: a decode step reads whole expert weight matrices to produce one output row, so memory is its roofline and the compute peak is the wrong denominator. The prefill cases stay on gflops." }, "vars": { "tokens": 1, "hidden": 2048, "experts": 16, "inter": 2048, "topK": 8 }, "attrs": { "k": 8, "activation_type": "swiglu", "swiglu_fusion": 2, "normalize_routing_weights": 1 }, "inputs": { "inputT": { "shape": [1, 2048], "dtype": "float32", "dist": "normal", "seed": 3031, "scale": 0.5 }, "routerT": { "shape": [1, 16], "dtype": "float32", "dist": "normal", "seed": 3032, "scale": 0.7 }, "fc1T": { "shape": [16, 4096, 2048], "dtype": "float32", "dist": "normal", "seed": 3033, "scale": 0.05 }, "fc2T": { "shape": [16, 2048, 2048], "dtype": "float32", "dist": "normal", "seed": 3034, "scale": 0.05 } }, "outputs": { "outputT": { "shape": [1, 2048], "dtype": "float32" } }, "bench": { "metrics": [{ "type": "bandwidth", "value": "args.topK * 3 * args.hidden * args.inter * 4" }] } }, { "name": "moe-f32-olmoe-prefill-t128-h2048-e16-k8-i2048", "preset": "model", "provenance": { "notes": "OLMoE class defaults at a 128-token prefill chunk." }, "vars": { "tokens": 128, "hidden": 2048, "experts": 16, "inter": 2048, "topK": 8 }, "attrs": { "k": 8, "activation_type": "swiglu", "swiglu_fusion": 2, "normalize_routing_weights": 1 }, "inputs": { "inputT": { "shape": [128, 2048], "dtype": "float32", "dist": "normal", "seed": 3041, "scale": 0.5 }, "routerT": { "shape": [128, 16], "dtype": "float32", "dist": "normal", "seed": 3042, "scale": 0.7 }, "fc1T": { "shape": [16, 4096, 2048], "dtype": "float32", "dist": "normal", "seed": 3043, "scale": 0.05 }, "fc2T": { "shape": [16, 2048, 2048], "dtype": "float32", "dist": "normal", "seed": 3044, "scale": 0.05 } }, "outputs": { "outputT": { "shape": [128, 2048], "dtype": "float32" } }, "bench": { "metrics": [{ "type": "gflops", "value": "2 * args.tokens * args.topK * args.hidden * args.inter * 3" }] } }, { "name": "moe-f32-qwen2-moe-decode-t1-h2048-e20-k4-i1408", "preset": "model", "provenance": { "notes": "Qwen2-MoE class defaults (hidden_size 2048, moe_intermediate_size 1408, num_experts_per_tok 4); expert count capped at 20 of the config's 60. Scored as bandwidth, not gflops: a decode step reads whole expert weight matrices to produce one output row, so memory is its roofline and the compute peak is the wrong denominator. The prefill cases stay on gflops." }, "vars": { "tokens": 1, "hidden": 2048, "experts": 20, "inter": 1408, "topK": 4 }, "attrs": { "k": 4, "activation_type": "swiglu", "swiglu_fusion": 2, "normalize_routing_weights": 1 }, "inputs": { "inputT": { "shape": [1, 2048], "dtype": "float32", "dist": "normal", "seed": 3051, "scale": 0.5 }, "routerT": { "shape": [1, 20], "dtype": "float32", "dist": "normal", "seed": 3052, "scale": 0.7 }, "fc1T": { "shape": [20, 2816, 2048], "dtype": "float32", "dist": "normal", "seed": 3053, "scale": 0.05 }, "fc2T": { "shape": [20, 2048, 1408], "dtype": "float32", "dist": "normal", "seed": 3054, "scale": 0.05 } }, "outputs": { "outputT": { "shape": [1, 2048], "dtype": "float32" } }, "bench": { "metrics": [{ "type": "bandwidth", "value": "args.topK * 3 * args.hidden * args.inter * 4" }] } }, { "name": "moe-f32-qwen2-moe-prefill-t256-h2048-e20-k4-i1408", "preset": "model", "provenance": { "notes": "Qwen2-MoE class defaults at a 256-token prefill chunk." }, "vars": { "tokens": 256, "hidden": 2048, "experts": 20, "inter": 1408, "topK": 4 }, "attrs": { "k": 4, "activation_type": "swiglu", "swiglu_fusion": 2, "normalize_routing_weights": 1 }, "inputs": { "inputT": { "shape": [256, 2048], "dtype": "float32", "dist": "normal", "seed": 3061, "scale": 0.5 }, "routerT": { "shape": [256, 20], "dtype": "float32", "dist": "normal", "seed": 3062, "scale": 0.7 }, "fc1T": { "shape": [20, 2816, 2048], "dtype": "float32", "dist": "normal", "seed": 3063, "scale": 0.05 }, "fc2T": { "shape": [20, 2048, 1408], "dtype": "float32", "dist": "normal", "seed": 3064, "scale": 0.05 } }, "outputs": { "outputT": { "shape": [256, 2048], "dtype": "float32" } }, "bench": { "metrics": [{ "type": "gflops", "value": "2 * args.tokens * args.topK * args.hidden * args.inter * 3" }] } } ] }