| { |
| "op": "com.microsoft.MultiHeadAttention", |
| "cases": [ |
| { |
| "name": "minilm-self-attention-b1-s20-h12-d32", |
| "provenance": { |
| "notes": "Upper boundary of the whole-head small-sequence route at 12 heads and sequence length 20. Paired with the sequence-24 case, it checks that the mutually exclusive route predicates meet without a gap." |
| }, |
| "vars": { "batch": 1, "qSeq": 20, "kvSeq": 20, "heads": 12, "headDim": 32 }, |
| "attrs": { "num_heads": 12 }, |
| "inputs": { |
| "queryT": { "shape": [1, 20, 384], "dtype": "float32", "dist": "normal", "seed": 9251, "scale": 0.2 }, |
| "keyT": { "shape": [1, 20, 384], "dtype": "float32", "dist": "normal", "seed": 9252, "scale": 0.2 }, |
| "valueT": { "shape": [1, 20, 384], "dtype": "float32", "dist": "normal", "seed": 9253, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 20, 384], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } |
| ] |
| } |
| }, |
| { |
| "name": "minilm-self-attention-b1-s24-h12-d32", |
| "provenance": { |
| "notes": "Boundary immediately above the whole-head small-sequence route. This shape satisfies flashShapeOk, so it checks that the mutually exclusive route predicates remain aligned." |
| }, |
| "vars": { "batch": 1, "qSeq": 24, "kvSeq": 24, "heads": 12, "headDim": 32 }, |
| "attrs": { "num_heads": 12 }, |
| "inputs": { |
| "queryT": { "shape": [1, 24, 384], "dtype": "float32", "dist": "normal", "seed": 9261, "scale": 0.2 }, |
| "keyT": { "shape": [1, 24, 384], "dtype": "float32", "dist": "normal", "seed": 9262, "scale": 0.2 }, |
| "valueT": { "shape": [1, 24, 384], "dtype": "float32", "dist": "normal", "seed": 9263, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 24, 384], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } |
| ] |
| } |
| }, |
| { |
| "name": "minilm-self-attention-b1-s11-h12-d32", |
| "provenance": { |
| "notes": "all-MiniLM-L6-v2 self-attention at a short sentence: 12 heads of 32, bidirectional, no bias." |
| }, |
| "vars": { "batch": 1, "qSeq": 11, "kvSeq": 11, "heads": 12, "headDim": 32 }, |
| "attrs": { "num_heads": 12 }, |
| "inputs": { |
| "queryT": { "shape": [1, 11, 384], "dtype": "float32", "dist": "normal", "seed": 9201, "scale": 0.2 }, |
| "keyT": { "shape": [1, 11, 384], "dtype": "float32", "dist": "normal", "seed": 9202, "scale": 0.2 }, |
| "valueT": { "shape": [1, 11, 384], "dtype": "float32", "dist": "normal", "seed": 9203, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 11, 384], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } |
| ] |
| } |
| }, |
| { |
| "name": "minilm-self-attention-b1-s32-h12-d32", |
| "provenance": { "notes": "all-MiniLM-L6-v2 self-attention: 12 heads of 32, bidirectional, no bias." }, |
| "vars": { "batch": 1, "qSeq": 32, "kvSeq": 32, "heads": 12, "headDim": 32 }, |
| "attrs": { "num_heads": 12 }, |
| "inputs": { |
| "queryT": { "shape": [1, 32, 384], "dtype": "float32", "dist": "normal", "seed": 9211, "scale": 0.2 }, |
| "keyT": { "shape": [1, 32, 384], "dtype": "float32", "dist": "normal", "seed": 9212, "scale": 0.2 }, |
| "valueT": { "shape": [1, 32, 384], "dtype": "float32", "dist": "normal", "seed": 9213, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 32, 384], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } |
| ] |
| } |
| }, |
| { |
| "name": "minilm-self-attention-b1-s128-h12-d32", |
| "provenance": { "notes": "all-MiniLM-L6-v2 self-attention: 12 heads of 32, bidirectional, no bias." }, |
| "vars": { "batch": 1, "qSeq": 128, "kvSeq": 128, "heads": 12, "headDim": 32 }, |
| "attrs": { "num_heads": 12 }, |
| "inputs": { |
| "queryT": { "shape": [1, 128, 384], "dtype": "float32", "dist": "normal", "seed": 9221, "scale": 0.2 }, |
| "keyT": { "shape": [1, 128, 384], "dtype": "float32", "dist": "normal", "seed": 9222, "scale": 0.2 }, |
| "valueT": { "shape": [1, 128, 384], "dtype": "float32", "dist": "normal", "seed": 9223, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 128, 384], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } |
| ] |
| } |
| }, |
| { |
| "name": "mha-f32-1x64x64-h4d16", |
| "preset": "smoke", |
| "vars": { "batch": 1, "qSeq": 64, "kvSeq": 64, "heads": 4, "headDim": 16 }, |
| "attrs": { "num_heads": 4, "scale": 0.25 }, |
| "inputs": { |
| "queryT": { "shape": [1, 64, 64], "dtype": "float32", "dist": "normal", "seed": 203, "scale": 0.2 }, |
| "keyT": { "shape": [1, 64, 64], "dtype": "float32", "dist": "normal", "seed": 204, "scale": 0.2 }, |
| "valueT": { "shape": [1, 64, 64], "dtype": "float32", "dist": "normal", "seed": 205, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 64, 64], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } |
| ] |
| } |
| }, |
| { |
| "name": "mha-f32-bias-causal-1x64x64-h4d16", |
| "preset": "smoke", |
| "vars": { "batch": 1, "qSeq": 64, "kvSeq": 64, "heads": 4, "headDim": 16 }, |
| "attrs": { "num_heads": 4, "scale": 0.25, "unidirectional": 1 }, |
| "inputs": { |
| "queryT": { "shape": [1, 64, 64], "dtype": "float32", "dist": "normal", "seed": 204, "scale": 0.2 }, |
| "keyT": { "shape": [1, 64, 64], "dtype": "float32", "dist": "normal", "seed": 205, "scale": 0.2 }, |
| "valueT": { "shape": [1, 64, 64], "dtype": "float32", "dist": "normal", "seed": 206, "scale": 0.2 }, |
| "biasT": { "shape": [192], "dtype": "float32", "dist": "normal", "seed": 207, "scale": 0.05 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 64, 64], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.heads * args.headDim * args.qSeq * (args.qSeq + 1) / 2" } |
| ] |
| } |
| }, |
| { |
| "name": "decode-h8-d64-kv512", |
| "preset": "smoke", |
| "vars": { "batch": 1, "qSeq": 1, "kvSeq": 512, "heads": 8, "headDim": 64 }, |
| "attrs": { "num_heads": 8, "scale": 0.125 }, |
| "inputs": { |
| "queryT": { "shape": [1, 1, 512], "dtype": "float32", "dist": "normal", "seed": 467, "scale": 0.2 }, |
| "keyT": { "shape": [1, 512, 512], "dtype": "float32", "dist": "normal", "seed": 468, "scale": 0.2 }, |
| "valueT": { "shape": [1, 512, 512], "dtype": "float32", "dist": "normal", "seed": 469, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 1, 512], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim) * 4" |
| } |
| ] |
| } |
| }, |
| { |
| "name": "decode-h8-d64-kv512-bias", |
| "preset": "smoke", |
| "vars": { "batch": 1, "qSeq": 1, "kvSeq": 512, "heads": 8, "headDim": 64 }, |
| "attrs": { "num_heads": 8, "scale": 0.125 }, |
| "inputs": { |
| "queryT": { "shape": [1, 1, 512], "dtype": "float32", "dist": "normal", "seed": 471, "scale": 0.2 }, |
| "keyT": { "shape": [1, 512, 512], "dtype": "float32", "dist": "normal", "seed": 472, "scale": 0.2 }, |
| "valueT": { "shape": [1, 512, 512], "dtype": "float32", "dist": "normal", "seed": 473, "scale": 0.2 }, |
| "biasT": { "shape": [1536], "dtype": "float32", "dist": "normal", "seed": 474, "scale": 0.05 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 1, 512], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim + 3 * args.heads * args.headDim) * 4" |
| } |
| ] |
| } |
| }, |
| { |
| "name": "decode-h8-d64-kv768", |
| "preset": "smoke", |
| "vars": { "batch": 1, "qSeq": 1, "kvSeq": 768, "heads": 8, "headDim": 64 }, |
| "attrs": { "num_heads": 8, "scale": 0.125 }, |
| "inputs": { |
| "queryT": { "shape": [1, 1, 512], "dtype": "float32", "dist": "normal", "seed": 477, "scale": 0.2 }, |
| "keyT": { "shape": [1, 768, 512], "dtype": "float32", "dist": "normal", "seed": 478, "scale": 0.2 }, |
| "valueT": { "shape": [1, 768, 512], "dtype": "float32", "dist": "normal", "seed": 479, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 1, 512], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim) * 4" |
| } |
| ] |
| } |
| }, |
| { |
| "name": "decode-h8-d64-kv768-bias", |
| "preset": "smoke", |
| "vars": { "batch": 1, "qSeq": 1, "kvSeq": 768, "heads": 8, "headDim": 64 }, |
| "attrs": { "num_heads": 8, "scale": 0.125 }, |
| "inputs": { |
| "queryT": { "shape": [1, 1, 512], "dtype": "float32", "dist": "normal", "seed": 481, "scale": 0.2 }, |
| "keyT": { "shape": [1, 768, 512], "dtype": "float32", "dist": "normal", "seed": 482, "scale": 0.2 }, |
| "valueT": { "shape": [1, 768, 512], "dtype": "float32", "dist": "normal", "seed": 483, "scale": 0.2 }, |
| "biasT": { "shape": [1536], "dtype": "float32", "dist": "normal", "seed": 484, "scale": 0.05 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 1, 512], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim + 3 * args.heads * args.headDim) * 4" |
| } |
| ] |
| } |
| }, |
| { |
| "name": "decode-h4-d64-kv512", |
| "preset": "smoke", |
| "vars": { "batch": 1, "qSeq": 1, "kvSeq": 512, "heads": 4, "headDim": 64 }, |
| "attrs": { "num_heads": 4, "scale": 0.125 }, |
| "inputs": { |
| "queryT": { "shape": [1, 1, 256], "dtype": "float32", "dist": "normal", "seed": 367, "scale": 0.2 }, |
| "keyT": { "shape": [1, 512, 256], "dtype": "float32", "dist": "normal", "seed": 368, "scale": 0.2 }, |
| "valueT": { "shape": [1, 512, 256], "dtype": "float32", "dist": "normal", "seed": 369, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 1, 256], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim) * 4" |
| } |
| ] |
| } |
| }, |
| { |
| "name": "decode-h4-d64-kv512-bias", |
| "preset": "smoke", |
| "vars": { "batch": 1, "qSeq": 1, "kvSeq": 512, "heads": 4, "headDim": 64 }, |
| "attrs": { "num_heads": 4, "scale": 0.125 }, |
| "inputs": { |
| "queryT": { "shape": [1, 1, 256], "dtype": "float32", "dist": "normal", "seed": 371, "scale": 0.2 }, |
| "keyT": { "shape": [1, 512, 256], "dtype": "float32", "dist": "normal", "seed": 372, "scale": 0.2 }, |
| "valueT": { "shape": [1, 512, 256], "dtype": "float32", "dist": "normal", "seed": 373, "scale": 0.2 }, |
| "biasT": { "shape": [768], "dtype": "float32", "dist": "normal", "seed": 374, "scale": 0.05 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 1, 256], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim + 3 * args.heads * args.headDim) * 4" |
| } |
| ] |
| } |
| }, |
| { |
| "name": "decode-h8-d128-kv2048", |
| "preset": "model", |
| "vars": { "batch": 1, "qSeq": 1, "kvSeq": 2048, "heads": 8, "headDim": 128 }, |
| "attrs": { "num_heads": 8, "scale": 0.08838834764831845 }, |
| "inputs": { |
| "queryT": { "shape": [1, 1, 1024], "dtype": "float32", "dist": "normal", "seed": 1367, "scale": 0.2 }, |
| "keyT": { "shape": [1, 2048, 1024], "dtype": "float32", "dist": "normal", "seed": 1368, "scale": 0.2 }, |
| "valueT": { "shape": [1, 2048, 1024], "dtype": "float32", "dist": "normal", "seed": 1369, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 1, 1024], "dtype": "float32" } }, |
| "bench": { |
| "primary": true, |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim) * 4" |
| } |
| ] |
| } |
| }, |
| { |
| "name": "flux2-joint-attn-img512-s1536-24h-d128", |
| "preset": "model", |
| "vars": { "batch": 1, "qSeq": 1536, "kvSeq": 1536, "heads": 24, "headDim": 128 }, |
| "attrs": { "num_heads": 24, "scale": 0.08838834764831845 }, |
| "inputs": { |
| "queryT": { "shape": [1, 1536, 3072], "dtype": "float32", "dist": "normal", "seed": 360, "scale": 0.2 }, |
| "keyT": { "shape": [1, 1536, 3072], "dtype": "float32", "dist": "normal", "seed": 361, "scale": 0.2 }, |
| "valueT": { "shape": [1, 1536, 3072], "dtype": "float32", "dist": "normal", "seed": 362, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 1536, 3072], "dtype": "float32" } }, |
| "bench": { |
| "primary": true, |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } |
| ] |
| }, |
| "tunableSpace": { "MATERIALIZED_FUSED_SOFTMAX_MIN_SCORE_BYTES": [16777216, 268435456] } |
| }, |
| { |
| "name": "encoder-prefill-q1500-kv1500-h6-d64-tail", |
| "preset": "model", |
| "vars": { "batch": 1, "qSeq": 1500, "kvSeq": 1500, "heads": 6, "headDim": 64 }, |
| "attrs": { "num_heads": 6, "scale": 0.125 }, |
| "tunableSpace": { |
| "MATERIALIZED_SGMAT_QUERY_TILE": [16, 32, 64], |
| "MATERIALIZED_SGMAT_KEY_TILE": [32, 64], |
| "MATERIALIZED_FUSED_SOFTMAX_MIN_SCORE_BYTES": [16777216, 268435456] |
| }, |
| "inputs": { |
| "queryT": { "shape": [1, 1500, 384], "dtype": "float32", "dist": "normal", "seed": 460, "scale": 0.2 }, |
| "keyT": { "shape": [1, 1500, 384], "dtype": "float32", "dist": "normal", "seed": 461, "scale": 0.2 }, |
| "valueT": { "shape": [1, 1500, 384], "dtype": "float32", "dist": "normal", "seed": 462, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 1500, 384], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } |
| ] |
| } |
| }, |
| { |
| "name": "decode-cross-q1-kv1500-h6-d64-underfilled-wide-subgroup", |
| "preset": "model", |
| "vars": { "batch": 1, "qSeq": 1, "kvSeq": 1500, "heads": 6, "headDim": 64 }, |
| "attrs": { "num_heads": 6 }, |
| "tunableSpace": { |
| "WORKGROUP_SIZE": [64, 128, 256], |
| "DECODE_MAX_SPLITS": [4, 6, 8, 10, 12, 16], |
| "DECODE_KEYS_PER_SPLIT": [128] |
| }, |
| "inputs": { |
| "queryT": { "shape": [1, 1, 384], "dtype": "float32", "dist": "normal", "seed": 463, "scale": 0.2 }, |
| "keyT": { "shape": [1, 1500, 384], "dtype": "float32", "dist": "normal", "seed": 464, "scale": 0.2 }, |
| "valueT": { "shape": [1, 1500, 384], "dtype": "float32", "dist": "normal", "seed": 465, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 1, 384], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim) * 4" |
| } |
| ] |
| } |
| }, |
| { |
| "name": "flux2-joint-attn-img512-s1536-24h-d128-bias", |
| "preset": "model", |
| "vars": { "batch": 1, "qSeq": 1536, "kvSeq": 1536, "heads": 24, "headDim": 128 }, |
| "attrs": { "num_heads": 24, "scale": 0.08838834764831845 }, |
| "inputs": { |
| "queryT": { "shape": [1, 1536, 3072], "dtype": "float32", "dist": "normal", "seed": 363, "scale": 0.2 }, |
| "keyT": { "shape": [1, 1536, 3072], "dtype": "float32", "dist": "normal", "seed": 364, "scale": 0.2 }, |
| "valueT": { "shape": [1, 1536, 3072], "dtype": "float32", "dist": "normal", "seed": 365, "scale": 0.2 }, |
| "biasT": { "shape": [9216], "dtype": "float32", "dist": "normal", "seed": 366, "scale": 0.05 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 1536, 3072], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } |
| ] |
| } |
| }, |
| { |
| "name": "flux2-joint-attn-img512-s1536-24h-d128-f16", |
| "preset": "model", |
| "vars": { "batch": 1, "qSeq": 1536, "kvSeq": 1536, "heads": 24, "headDim": 128 }, |
| "attrs": { "num_heads": 24, "scale": 0.08838834764831845 }, |
| "inputs": { |
| "queryT": { "shape": [1, 1536, 3072], "dtype": "float16", "dist": "normal", "seed": 360, "scale": 0.2 }, |
| "keyT": { "shape": [1, 1536, 3072], "dtype": "float16", "dist": "normal", "seed": 361, "scale": 0.2 }, |
| "valueT": { "shape": [1, 1536, 3072], "dtype": "float16", "dist": "normal", "seed": 362, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 1536, 3072], "dtype": "float16" } }, |
| "bench": { |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } |
| ] |
| } |
| }, |
| { |
| "name": "decode-h8-d128-kv2048-f16", |
| "preset": "model", |
| "vars": { "batch": 1, "qSeq": 1, "kvSeq": 2048, "heads": 8, "headDim": 128 }, |
| "attrs": { "num_heads": 8, "scale": 0.08838834764831845 }, |
| "inputs": { |
| "queryT": { "shape": [1, 1, 1024], "dtype": "float16", "dist": "normal", "seed": 1367, "scale": 0.2 }, |
| "keyT": { "shape": [1, 2048, 1024], "dtype": "float16", "dist": "normal", "seed": 1368, "scale": 0.2 }, |
| "valueT": { "shape": [1, 2048, 1024], "dtype": "float16", "dist": "normal", "seed": 1369, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 1, 1024], "dtype": "float16" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim) * 2" |
| } |
| ] |
| } |
| }, |
| { |
| "name": "prefill-lpq4-d80-s512-h8-BLINDSPOT", |
| "preset": "smoke", |
| "vars": { "batch": 1, "qSeq": 512, "kvSeq": 512, "heads": 8, "headDim": 80 }, |
| "attrs": { "num_heads": 8, "scale": 0.111803398875 }, |
| "inputs": { |
| "queryT": { "shape": [1, 512, 640], "dtype": "float32", "dist": "normal", "seed": 5101, "scale": 0.2 }, |
| "keyT": { "shape": [1, 512, 640], "dtype": "float32", "dist": "normal", "seed": 5102, "scale": 0.2 }, |
| "valueT": { "shape": [1, 512, 640], "dtype": "float32", "dist": "normal", "seed": 5103, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 512, 640], "dtype": "float32" } }, |
| "bench": { |
| "primary": true, |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } |
| ] |
| } |
| }, |
| { |
| "name": "prefill-cluster-d128-s512-h8-HEALTHY", |
| "preset": "smoke", |
| "vars": { "batch": 1, "qSeq": 512, "kvSeq": 512, "heads": 8, "headDim": 128 }, |
| "attrs": { "num_heads": 8, "scale": 0.08838834764831845 }, |
| "inputs": { |
| "queryT": { "shape": [1, 512, 1024], "dtype": "float32", "dist": "normal", "seed": 5111, "scale": 0.2 }, |
| "keyT": { "shape": [1, 512, 1024], "dtype": "float32", "dist": "normal", "seed": 5112, "scale": 0.2 }, |
| "valueT": { "shape": [1, 512, 1024], "dtype": "float32", "dist": "normal", "seed": 5113, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 512, 1024], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } |
| ] |
| } |
| }, |
| { |
| "name": "prefill-online-noncluster-qseq16-h16-d64-FALLOFF", |
| "preset": "smoke", |
| "vars": { "batch": 1, "qSeq": 16, "kvSeq": 512, "heads": 16, "headDim": 64 }, |
| "attrs": { "num_heads": 16, "scale": 0.125 }, |
| "inputs": { |
| "queryT": { "shape": [1, 16, 1024], "dtype": "float32", "dist": "normal", "seed": 5121, "scale": 0.2 }, |
| "keyT": { "shape": [1, 512, 1024], "dtype": "float32", "dist": "normal", "seed": 5122, "scale": 0.2 }, |
| "valueT": { "shape": [1, 512, 1024], "dtype": "float32", "dist": "normal", "seed": 5123, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 16, 1024], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } |
| ] |
| } |
| }, |
| { |
| "name": "prefill-cluster-qseq64-h16-d64-HEALTHY", |
| "preset": "smoke", |
| "vars": { "batch": 1, "qSeq": 64, "kvSeq": 512, "heads": 16, "headDim": 64 }, |
| "attrs": { "num_heads": 16, "scale": 0.125 }, |
| "inputs": { |
| "queryT": { "shape": [1, 64, 1024], "dtype": "float32", "dist": "normal", "seed": 5131, "scale": 0.2 }, |
| "keyT": { "shape": [1, 512, 1024], "dtype": "float32", "dist": "normal", "seed": 5132, "scale": 0.2 }, |
| "valueT": { "shape": [1, 512, 1024], "dtype": "float32", "dist": "normal", "seed": 5133, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 64, 1024], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } |
| ] |
| } |
| }, |
| { |
| "name": "decode-splitk-longkv-kv16384-h4-d64-LONGCONTEXT", |
| "preset": "model", |
| "vars": { "batch": 1, "qSeq": 1, "kvSeq": 16384, "heads": 4, "headDim": 64 }, |
| "attrs": { "num_heads": 4, "scale": 0.125 }, |
| "inputs": { |
| "queryT": { "shape": [1, 1, 256], "dtype": "float32", "dist": "normal", "seed": 5141, "scale": 0.2 }, |
| "keyT": { "shape": [1, 16384, 256], "dtype": "float32", "dist": "normal", "seed": 5142, "scale": 0.2 }, |
| "valueT": { "shape": [1, 16384, 256], "dtype": "float32", "dist": "normal", "seed": 5143, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 1, 256], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim) * 4" |
| } |
| ] |
| } |
| }, |
| { |
| "name": "scalar-fallback-odd-headdim6-prefill-h8-q128-kv2048-SLOW", |
| "preset": "stress", |
| "vars": { "batch": 1, "qSeq": 128, "kvSeq": 2048, "heads": 8, "headDim": 6 }, |
| "attrs": { "num_heads": 8, "scale": 0.408248290463863 }, |
| "inputs": { |
| "queryT": { "shape": [1, 128, 48], "dtype": "float32", "dist": "normal", "seed": 6201, "scale": 0.2 }, |
| "keyT": { "shape": [1, 2048, 48], "dtype": "float32", "dist": "normal", "seed": 6202, "scale": 0.2 }, |
| "valueT": { "shape": [1, 2048, 48], "dtype": "float32", "dist": "normal", "seed": 6203, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 128, 48], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } |
| ] |
| } |
| }, |
| { |
| "name": "scalar-fallback-shortq16-longkv4096-d128-h4-b2-SLOW", |
| "preset": "stress", |
| "vars": { "batch": 2, "qSeq": 16, "kvSeq": 4096, "heads": 4, "headDim": 128 }, |
| "attrs": { "num_heads": 4, "scale": 0.08838834764831845 }, |
| "inputs": { |
| "queryT": { "shape": [2, 16, 512], "dtype": "float32", "dist": "normal", "seed": 6211, "scale": 0.2 }, |
| "keyT": { "shape": [2, 4096, 512], "dtype": "float32", "dist": "normal", "seed": 6212, "scale": 0.2 }, |
| "valueT": { "shape": [2, 4096, 512], "dtype": "float32", "dist": "normal", "seed": 6213, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [2, 16, 512], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } |
| ] |
| } |
| }, |
| { |
| "name": "mha-f32-q31-kv511-h8-d64-double-threshold-cliff", |
| "preset": "stress", |
| "vars": { "batch": 1, "qSeq": 31, "kvSeq": 511, "heads": 8, "headDim": 64 }, |
| "attrs": { "num_heads": 8, "scale": 0.125 }, |
| "inputs": { |
| "queryT": { "shape": [1, 31, 512], "dtype": "float32", "dist": "normal", "seed": 6221, "scale": 0.2 }, |
| "keyT": { "shape": [1, 511, 512], "dtype": "float32", "dist": "normal", "seed": 6222, "scale": 0.2 }, |
| "valueT": { "shape": [1, 511, 512], "dtype": "float32", "dist": "normal", "seed": 6223, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 31, 512], "dtype": "float32", "dist": "empty" } }, |
| "bench": { |
| "primary": true, |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } |
| ] |
| } |
| }, |
| { |
| "name": "mha-f32-q32-kv512-h8-d64-threshold-control", |
| "preset": "stress", |
| "vars": { "batch": 1, "qSeq": 32, "kvSeq": 512, "heads": 8, "headDim": 64 }, |
| "attrs": { "num_heads": 8, "scale": 0.125 }, |
| "inputs": { |
| "queryT": { "shape": [1, 32, 512], "dtype": "float32", "dist": "normal", "seed": 6224, "scale": 0.2 }, |
| "keyT": { "shape": [1, 512, 512], "dtype": "float32", "dist": "normal", "seed": 6225, "scale": 0.2 }, |
| "valueT": { "shape": [1, 512, 512], "dtype": "float32", "dist": "normal", "seed": 6226, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 32, 512], "dtype": "float32", "dist": "empty" } }, |
| "bench": { |
| "primary": true, |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } |
| ] |
| } |
| }, |
| { |
| "name": "materialized-sgmat-q768-kv772-h4-d80-general-tails", |
| "preset": "model", |
| "vars": { "batch": 1, "qSeq": 768, "kvSeq": 772, "heads": 4, "headDim": 80 }, |
| "attrs": { "num_heads": 4, "scale": 0.11180339887498948 }, |
| "inputs": { |
| "queryT": { "shape": [1, 768, 320], "dtype": "float32", "dist": "normal", "seed": 6331, "scale": 0.2 }, |
| "keyT": { "shape": [1, 772, 320], "dtype": "float32", "dist": "normal", "seed": 6332, "scale": 0.2 }, |
| "valueT": { "shape": [1, 772, 320], "dtype": "float32", "dist": "normal", "seed": 6333, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 768, 320], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } |
| ] |
| } |
| }, |
| { |
| "name": "materialized-portable-cached-q512-kv512-h1-d128", |
| "preset": "model", |
| "vars": { "batch": 1, "qSeq": 512, "kvSeq": 512, "heads": 1, "headDim": 128 }, |
| "attrs": { "num_heads": 1, "scale": 0.08838834764831845 }, |
| "tunableSpace": { "MATERIALIZED_CACHED_SOFTMAX_WORKGROUP_SIZE": [64, 128, 256] }, |
| "inputs": { |
| "queryT": { "shape": [1, 512, 128], "dtype": "float32", "dist": "normal", "seed": 6341, "scale": 0.2 }, |
| "keyT": { "shape": [1, 512, 128], "dtype": "float32", "dist": "normal", "seed": 6342, "scale": 0.2 }, |
| "valueT": { "shape": [1, 512, 128], "dtype": "float32", "dist": "normal", "seed": 6343, "scale": 0.2 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 512, 128], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } |
| ] |
| } |
| }, |
| { |
| "name": "materialized-portable-cached-q512-kv512-h1-d64-bias", |
| "preset": "model", |
| "vars": { "batch": 1, "qSeq": 512, "kvSeq": 512, "heads": 1, "headDim": 64 }, |
| "attrs": { "num_heads": 1, "scale": 0.125 }, |
| "tunableSpace": { "MATERIALIZED_CACHED_SOFTMAX_WORKGROUP_SIZE": [64, 128, 256] }, |
| "inputs": { |
| "queryT": { "shape": [1, 512, 64], "dtype": "float32", "dist": "normal", "seed": 6351, "scale": 0.2 }, |
| "keyT": { "shape": [1, 512, 64], "dtype": "float32", "dist": "normal", "seed": 6352, "scale": 0.2 }, |
| "valueT": { "shape": [1, 512, 64], "dtype": "float32", "dist": "normal", "seed": 6353, "scale": 0.2 }, |
| "biasT": { "shape": [192], "dtype": "float32", "dist": "normal", "seed": 6354, "scale": 0.05 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 512, 64], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } |
| ] |
| } |
| }, |
| { |
| "name": "prefill-f32-bias-q128-kv128-h4-d64", |
| "preset": "model", |
| "provenance": { |
| "notes": "Lower f32 fused-bias register-topology boundary comparing the Q32-broadcast and subgroup-cluster routes." |
| }, |
| "vars": { "batch": 1, "qSeq": 128, "kvSeq": 128, "heads": 4, "headDim": 64 }, |
| "attrs": { "num_heads": 4, "scale": 0.125 }, |
| "inputs": { |
| "queryT": { "shape": [1, 128, 256], "dtype": "float32", "dist": "normal", "seed": 6355, "scale": 0.2 }, |
| "keyT": { "shape": [1, 128, 256], "dtype": "float32", "dist": "normal", "seed": 6356, "scale": 0.2 }, |
| "valueT": { "shape": [1, 128, 256], "dtype": "float32", "dist": "normal", "seed": 6357, "scale": 0.2 }, |
| "biasT": { "shape": [768], "dtype": "float32", "dist": "normal", "seed": 6358, "scale": 0.05 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 128, 256], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } |
| ] |
| } |
| }, |
| { |
| "name": "prefill-f32-bias-q128-kv128-h4-d256", |
| "preset": "model", |
| "provenance": { |
| "notes": "Upper f32 fused-bias register-topology boundary comparing the Q32-broadcast and subgroup-cluster routes." |
| }, |
| "vars": { "batch": 1, "qSeq": 128, "kvSeq": 128, "heads": 4, "headDim": 256 }, |
| "attrs": { "num_heads": 4, "scale": 0.0625 }, |
| "inputs": { |
| "queryT": { "shape": [1, 128, 1024], "dtype": "float32", "dist": "normal", "seed": 6361, "scale": 0.2 }, |
| "keyT": { "shape": [1, 128, 1024], "dtype": "float32", "dist": "normal", "seed": 6362, "scale": 0.2 }, |
| "valueT": { "shape": [1, 128, 1024], "dtype": "float32", "dist": "normal", "seed": 6363, "scale": 0.2 }, |
| "biasT": { "shape": [3072], "dtype": "float32", "dist": "normal", "seed": 6364, "scale": 0.05 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 128, 1024], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } |
| ] |
| } |
| }, |
| { |
| "name": "mha-f32-q32-kv512-h8-d64-attn-bias-flash", |
| "preset": "stress", |
| "vars": { "batch": 1, "qSeq": 32, "kvSeq": 512, "heads": 8, "headDim": 64 }, |
| "attrs": { "num_heads": 8, "scale": 0.125 }, |
| "inputs": { |
| "queryT": { "shape": [1, 32, 512], "dtype": "float32", "dist": "normal", "seed": 6224, "scale": 0.2 }, |
| "keyT": { "shape": [1, 512, 512], "dtype": "float32", "dist": "normal", "seed": 6225, "scale": 0.2 }, |
| "valueT": { "shape": [1, 512, 512], "dtype": "float32", "dist": "normal", "seed": 6226, "scale": 0.2 }, |
| "attentionBiasT": { "shape": [1, 1, 32, 512], "dtype": "float32", "dist": "normal", "seed": 471, "scale": 0.3 } |
| }, |
| "outputs": { "outputT": { "shape": [1, 32, 512], "dtype": "float32", "dist": "empty" } }, |
| "bench": { |
| "primary": true, |
| "metrics": [ |
| { "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" } |
| ] |
| } |
| } |
| ] |
| } |
|
|
