| { |
| "op": "com.microsoft.GatherBlockQuantized", |
| "tunableSpace": { "workgroupSize": [64, 128, 256] }, |
| "cases": [ |
| { |
| "name": "gather-block-q8-4096x1024-idx1024", |
| "preset": "smoke", |
| "vars": { "rows": 4096, "cols": 1024, "indexCount": 1024, "bits": 8, "blockSize": 32 }, |
| "attrs": { "bits": 8, "block_size": 32 }, |
| "inputs": { |
| "dataT": { "shape": [4096, 1024], "dtype": "uint8", "dist": "randint", "seed": 205, "min": 0, "max": 255 }, |
| "indicesT": { "shape": [1024], "dtype": "uint32", "dist": "linearMod", "seed": 205, "step": 37, "mod": 4096 }, |
| "scalesT": { |
| "shape": [4096, 32], |
| "dtype": "float32", |
| "dist": "uniform", |
| "seed": 206, |
| "offset": 0.04, |
| "scale": 0.01, |
| "signed": false |
| } |
| }, |
| "outputs": { "outputT": { "shape": [1024, 1024], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "value": "args.indexCount * (args.cols * args.bits / 8 + dim(shapes.scalesT, 1) * 4 + args.cols * 4 + 4)" |
| } |
| ] |
| } |
| }, |
| { |
| "name": "gather-block-q4-512x128-idx512", |
| "preset": "smoke", |
| "vars": { "rows": 512, "cols": 128, "indexCount": 512, "bits": 4, "blockSize": 32 }, |
| "attrs": { "bits": 4, "block_size": 32 }, |
| "inputs": { |
| "dataT": { "shape": [512, 64], "dtype": "uint8", "dist": "q4pair", "seed": 207 }, |
| "indicesT": { "shape": [512], "dtype": "uint32", "dist": "linearMod", "seed": 207, "step": 17, "mod": 512 }, |
| "scalesT": { |
| "shape": [512, 4], |
| "dtype": "float32", |
| "dist": "uniform", |
| "seed": 208, |
| "offset": 0.04, |
| "scale": 0.01, |
| "signed": false |
| } |
| }, |
| "outputs": { "outputT": { "shape": [512, 128], "dtype": "float32" } }, |
| "bench": { |
| "primary": true, |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "value": "args.indexCount * (args.cols * args.bits / 8 + dim(shapes.scalesT, 1) * 4 + args.cols * 4 + 4)" |
| } |
| ] |
| } |
| }, |
| { |
| "name": "gather-block-q4-zero-512x128-idx512", |
| "preset": "smoke", |
| "vars": { "rows": 512, "cols": 128, "indexCount": 512, "bits": 4, "blockSize": 32 }, |
| "attrs": { "bits": 4, "block_size": 32 }, |
| "inputs": { |
| "dataT": { "shape": [512, 64], "dtype": "uint8", "dist": "q4pair", "seed": 208 }, |
| "indicesT": { "shape": [512], "dtype": "uint32", "dist": "linearMod", "seed": 208, "step": 17, "mod": 512 }, |
| "scalesT": { |
| "shape": [512, 4], |
| "dtype": "float32", |
| "dist": "uniform", |
| "seed": 209, |
| "offset": 0.04, |
| "scale": 0.01, |
| "signed": false |
| }, |
| "zeroPointsT": { "shape": [512, 2], "dtype": "uint8", "dist": "q4pair", "seed": 210 } |
| }, |
| "outputs": { "outputT": { "shape": [512, 128], "dtype": "float32" } }, |
| "bench": { |
| "primary": true, |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "value": "args.indexCount * (args.cols * args.bits / 8 + dim(shapes.scalesT, 1) * 4 + dim(shapes.zeroPointsT, 1) + args.cols * 4 + 4)" |
| } |
| ] |
| } |
| }, |
| { |
| "name": "gather-block-q4-4096x1024-idx1024", |
| "preset": "smoke", |
| "vars": { "rows": 4096, "cols": 1024, "indexCount": 1024, "bits": 4, "blockSize": 32 }, |
| "attrs": { "bits": 4, "block_size": 32 }, |
| "inputs": { |
| "dataT": { "shape": [4096, 512], "dtype": "uint8", "dist": "q4pair", "seed": 211 }, |
| "indicesT": { "shape": [1024], "dtype": "uint32", "dist": "linearMod", "seed": 211, "step": 37, "mod": 4096 }, |
| "scalesT": { |
| "shape": [4096, 32], |
| "dtype": "float32", |
| "dist": "uniform", |
| "seed": 212, |
| "offset": 0.04, |
| "scale": 0.01, |
| "signed": false |
| } |
| }, |
| "outputs": { "outputT": { "shape": [1024, 1024], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "value": "args.indexCount * (args.cols * args.bits / 8 + dim(shapes.scalesT, 1) * 4 + args.cols * 4 + 4)" |
| } |
| ] |
| } |
| }, |
| { |
| "name": "gather-block-q4-zero-4096x1024-idx1024", |
| "preset": "smoke", |
| "vars": { "rows": 4096, "cols": 1024, "indexCount": 1024, "bits": 4, "blockSize": 32 }, |
| "attrs": { "bits": 4, "block_size": 32 }, |
| "inputs": { |
| "dataT": { "shape": [4096, 512], "dtype": "uint8", "dist": "q4pair", "seed": 212 }, |
| "indicesT": { "shape": [1024], "dtype": "uint32", "dist": "linearMod", "seed": 212, "step": 37, "mod": 4096 }, |
| "scalesT": { |
| "shape": [4096, 32], |
| "dtype": "float32", |
| "dist": "uniform", |
| "seed": 213, |
| "offset": 0.04, |
| "scale": 0.01, |
| "signed": false |
| }, |
| "zeroPointsT": { "shape": [4096, 16], "dtype": "uint8", "dist": "q4pair", "seed": 214 } |
| }, |
| "outputs": { "outputT": { "shape": [1024, 1024], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "value": "args.indexCount * (args.cols * args.bits / 8 + dim(shapes.scalesT, 1) * 4 + dim(shapes.zeroPointsT, 1) + args.cols * 4 + 4)" |
| } |
| ] |
| } |
| }, |
| { |
| "name": "gather-block-q8-alignment-cliff-cols1023-scalar-fallback", |
| "preset": "smoke", |
| "vars": { "rows": 4096, "cols": 1023, "indexCount": 1024, "bits": 8, "blockSize": 32 }, |
| "attrs": { "bits": 8, "block_size": 32 }, |
| "inputs": { |
| "dataT": { "shape": [4096, 1023], "dtype": "uint8", "dist": "randint", "seed": 215, "min": 0, "max": 255 }, |
| "indicesT": { "shape": [1024], "dtype": "uint32", "dist": "linearMod", "seed": 215, "step": 37, "mod": 4096 }, |
| "scalesT": { |
| "shape": [4096, 32], |
| "dtype": "float32", |
| "dist": "uniform", |
| "seed": 216, |
| "offset": 0.04, |
| "scale": 0.01, |
| "signed": false |
| } |
| }, |
| "outputs": { "outputT": { "shape": [1024, 1023], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "value": "args.indexCount * (args.cols * args.bits / 8 + dim(shapes.scalesT, 1) * 4 + args.cols * 4 + 4)" |
| } |
| ] |
| } |
| }, |
| { |
| "name": "gather-block-q8-alignment-healthy-cols1024-vec4", |
| "preset": "smoke", |
| "vars": { "rows": 4096, "cols": 1024, "indexCount": 1024, "bits": 8, "blockSize": 32 }, |
| "attrs": { "bits": 8, "block_size": 32 }, |
| "inputs": { |
| "dataT": { "shape": [4096, 1024], "dtype": "uint8", "dist": "randint", "seed": 215, "min": 0, "max": 255 }, |
| "indicesT": { "shape": [1024], "dtype": "uint32", "dist": "linearMod", "seed": 215, "step": 37, "mod": 4096 }, |
| "scalesT": { |
| "shape": [4096, 32], |
| "dtype": "float32", |
| "dist": "uniform", |
| "seed": 216, |
| "offset": 0.04, |
| "scale": 0.01, |
| "signed": false |
| } |
| }, |
| "outputs": { "outputT": { "shape": [1024, 1024], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "value": "args.indexCount * (args.cols * args.bits / 8 + dim(shapes.scalesT, 1) * 4 + args.cols * 4 + 4)" |
| } |
| ] |
| } |
| }, |
| { |
| "name": "gather-block-q8-dispatch-cliff-vocab-idx8192-cols4096-2dfold", |
| "preset": "stress", |
| "provenance": { |
| "notes": "Stress-only dispatch/capacity case: widened uint8 GPU storage brings the declared footprint to 392 MiB." |
| }, |
| "vars": { "rows": 16384, "cols": 4096, "indexCount": 8192, "bits": 8, "blockSize": 32 }, |
| "attrs": { "bits": 8, "block_size": 32 }, |
| "inputs": { |
| "dataT": { "shape": [16384, 4096], "dtype": "uint8", "dist": "randint", "seed": 217, "min": 0, "max": 255 }, |
| "indicesT": { "shape": [8192], "dtype": "uint32", "dist": "linearMod", "seed": 217, "step": 37, "mod": 16384 }, |
| "scalesT": { |
| "shape": [16384, 128], |
| "dtype": "float32", |
| "dist": "uniform", |
| "seed": 218, |
| "offset": 0.04, |
| "scale": 0.01, |
| "signed": false |
| } |
| }, |
| "outputs": { "outputT": { "shape": [8192, 4096], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "value": "args.indexCount * (args.cols * args.bits / 8 + dim(shapes.scalesT, 1) * 4 + args.cols * 4 + 4)" |
| } |
| ] |
| } |
| }, |
| { |
| "name": "gather-block-q8-dispatch-healthy-idx1024-cols4096-1d", |
| "preset": "stress", |
| "provenance": { |
| "notes": "Stress-only capacity sibling: widened uint8 GPU storage brings the declared footprint to 280 MiB." |
| }, |
| "vars": { "rows": 16384, "cols": 4096, "indexCount": 1024, "bits": 8, "blockSize": 32 }, |
| "attrs": { "bits": 8, "block_size": 32 }, |
| "inputs": { |
| "dataT": { "shape": [16384, 4096], "dtype": "uint8", "dist": "randint", "seed": 217, "min": 0, "max": 255 }, |
| "indicesT": { "shape": [1024], "dtype": "uint32", "dist": "linearMod", "seed": 217, "step": 37, "mod": 16384 }, |
| "scalesT": { |
| "shape": [16384, 128], |
| "dtype": "float32", |
| "dist": "uniform", |
| "seed": 218, |
| "offset": 0.04, |
| "scale": 0.01, |
| "signed": false |
| } |
| }, |
| "outputs": { "outputT": { "shape": [1024, 4096], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "value": "args.indexCount * (args.cols * args.bits / 8 + dim(shapes.scalesT, 1) * 4 + args.cols * 4 + 4)" |
| } |
| ] |
| } |
| }, |
| { |
| "name": "gather-block-q4-llama-v32000-h4096-idx1", |
| "preset": "model", |
| "provenance": { |
| "notes": "Llama class defaults (vocab_size 32000, hidden_size 4096) with a 4-bit block-quantized embedding table, gathering the single token of a decode step." |
| }, |
| "vars": { "rows": 32000, "cols": 4096, "indexCount": 1, "bits": 4, "blockSize": 128 }, |
| "attrs": { "bits": 4, "block_size": 128 }, |
| "inputs": { |
| "dataT": { "shape": [32000, 2048], "dtype": "uint8", "dist": "uniform", "seed": 6200, "min": 0, "max": 255 }, |
| "indicesT": { "shape": [1], "dtype": "uint32", "dist": "randint", "seed": 6201, "min": 0, "max": 31999 }, |
| "scalesT": { |
| "shape": [32000, 32], |
| "dtype": "float32", |
| "dist": "uniform", |
| "seed": 6202, |
| "min": 0.005, |
| "max": 0.05 |
| } |
| }, |
| "outputs": { "outputT": { "shape": [1, 4096], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "value": "args.indexCount * (args.cols * args.bits / 8 + dim(shapes.scalesT, 1) * 4 + args.cols * 4 + 4)" |
| } |
| ] |
| } |
| }, |
| { |
| "name": "gather-block-q4-llama-v32000-h4096-idx512", |
| "preset": "model", |
| "provenance": { "notes": "Llama class defaults gathering a 512-token prefill chunk." }, |
| "vars": { "rows": 32000, "cols": 4096, "indexCount": 512, "bits": 4, "blockSize": 128 }, |
| "attrs": { "bits": 4, "block_size": 128 }, |
| "inputs": { |
| "dataT": { "shape": [32000, 2048], "dtype": "uint8", "dist": "uniform", "seed": 6300, "min": 0, "max": 255 }, |
| "indicesT": { "shape": [512], "dtype": "uint32", "dist": "randint", "seed": 6301, "min": 0, "max": 31999 }, |
| "scalesT": { |
| "shape": [32000, 32], |
| "dtype": "float32", |
| "dist": "uniform", |
| "seed": 6302, |
| "min": 0.005, |
| "max": 0.05 |
| } |
| }, |
| "outputs": { "outputT": { "shape": [512, 4096], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "value": "args.indexCount * (args.cols * args.bits / 8 + dim(shapes.scalesT, 1) * 4 + args.cols * 4 + 4)" |
| } |
| ] |
| } |
| }, |
| { |
| "name": "gather-block-q4-qwen3-v151936-h2048-idx512", |
| "preset": "model", |
| "provenance": { |
| "notes": "Qwen3-MoE class defaults (vocab_size 151936, hidden_size 2048) -- the large-vocabulary case, where the gather is scattered over a much taller table." |
| }, |
| "vars": { "rows": 151936, "cols": 2048, "indexCount": 512, "bits": 4, "blockSize": 128 }, |
| "attrs": { "bits": 4, "block_size": 128 }, |
| "inputs": { |
| "dataT": { "shape": [151936, 1024], "dtype": "uint8", "dist": "uniform", "seed": 6400, "min": 0, "max": 255 }, |
| "indicesT": { "shape": [512], "dtype": "uint32", "dist": "randint", "seed": 6401, "min": 0, "max": 151935 }, |
| "scalesT": { |
| "shape": [151936, 16], |
| "dtype": "float32", |
| "dist": "uniform", |
| "seed": 6402, |
| "min": 0.005, |
| "max": 0.05 |
| } |
| }, |
| "outputs": { "outputT": { "shape": [512, 2048], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "value": "args.indexCount * (args.cols * args.bits / 8 + dim(shapes.scalesT, 1) * 4 + args.cols * 4 + 4)" |
| } |
| ] |
| } |
| }, |
| { |
| "name": "gather-block-q8-whisper-v51865-h384-idx448", |
| "preset": "model", |
| "provenance": { |
| "notes": "Whisper class defaults (vocab_size 51865, d_model 384, max_target_positions 448) with an 8-bit table, gathering a full decoder context." |
| }, |
| "vars": { "rows": 51865, "cols": 384, "indexCount": 448, "bits": 8, "blockSize": 32 }, |
| "attrs": { "bits": 8, "block_size": 32 }, |
| "inputs": { |
| "dataT": { "shape": [51865, 384], "dtype": "uint8", "dist": "uniform", "seed": 6500, "min": 0, "max": 255 }, |
| "indicesT": { "shape": [448], "dtype": "uint32", "dist": "randint", "seed": 6501, "min": 0, "max": 51864 }, |
| "scalesT": { |
| "shape": [51865, 12], |
| "dtype": "float32", |
| "dist": "uniform", |
| "seed": 6502, |
| "min": 0.005, |
| "max": 0.05 |
| } |
| }, |
| "outputs": { "outputT": { "shape": [448, 384], "dtype": "float32" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "value": "args.indexCount * (args.cols * args.bits / 8 + dim(shapes.scalesT, 1) * 4 + args.cols * 4 + 4)" |
| } |
| ] |
| } |
| } |
| ] |
| } |
|
|
