| { |
| "tunableSpace": { |
| "DP4A_TILE_M": [8, 16, 32, 64], |
| "DP4A_TILE_N": [32, 64, 128], |
| "DP4A_K_TILE_WORDS": [8, 16, 32], |
| "ATTEND_WORKGROUP": [32, 64, 128, 256] |
| }, |
| "cases": [ |
| { |
| "name": "qattention-bert-b1-s128", |
| "preset": "smoke", |
| "vars": { "batch": 1, "seq": 128, "inHidden": 768, "hidden": 768, "heads": 12, "past": 0 }, |
| "attrs": { "num_heads": 12 }, |
| "inputs": { |
| "input": { "dtype": "uint8", "shape": [1, 128, 768], "dist": "randint", "min": 0, "max": 256, "seed": 101 }, |
| "weight": { "dtype": "int8", "shape": [768, 2304], "dist": "randint", "min": -128, "max": 128, "seed": 102 }, |
| "bias": { "dtype": "float32", "shape": [2304], "dist": "normal", "scale": 0.3, "seed": 103 }, |
| "input_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.02 }, |
| "weight_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.012 }, |
| "mask_index": { "dtype": "int32", "shape": [1], "dist": "randint", "min": 1, "max": 129, "seed": 105 }, |
| "input_zero_point": { "dtype": "uint8", "shape": [1], "dist": "constant", "value": 128 }, |
| "weight_zero_point": { "dtype": "int8", "shape": [1], "dist": "constant", "value": 1 } |
| }, |
| "outputs": { "output": { "dtype": "float32", "shape": [1, 128, 768], "dist": "empty" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "gflops", |
| "name": "projection+attention", |
| "value": "2 * args.batch * args.seq * args.inHidden * 3 * args.hidden + 4 * args.batch * args.hidden * args.seq * (args.past + args.seq)" |
| }, |
| { |
| "type": "bandwidth", |
| "name": "operands", |
| "value": "args.inHidden * 3 * args.hidden * 4 + args.batch * args.seq * args.inHidden * 4 + args.batch * args.seq * args.hidden * 4" |
| } |
| ] |
| }, |
| "provenance": { "model": "BERT-base encoder: hidden 768, 12 heads, head size 64" } |
| }, |
| { |
| "name": "qattention-bert-b8-s128", |
| "preset": "smoke", |
| "vars": { "batch": 8, "seq": 128, "inHidden": 768, "hidden": 768, "heads": 12, "past": 0 }, |
| "attrs": { "num_heads": 12 }, |
| "inputs": { |
| "input": { "dtype": "uint8", "shape": [8, 128, 768], "dist": "randint", "min": 0, "max": 256, "seed": 101 }, |
| "weight": { "dtype": "int8", "shape": [768, 2304], "dist": "randint", "min": -128, "max": 128, "seed": 102 }, |
| "bias": { "dtype": "float32", "shape": [2304], "dist": "normal", "scale": 0.3, "seed": 103 }, |
| "input_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.02 }, |
| "weight_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.012 }, |
| "mask_index": { "dtype": "int32", "shape": [8], "dist": "randint", "min": 1, "max": 129, "seed": 105 }, |
| "input_zero_point": { "dtype": "uint8", "shape": [1], "dist": "constant", "value": 128 }, |
| "weight_zero_point": { "dtype": "int8", "shape": [1], "dist": "constant", "value": 1 } |
| }, |
| "outputs": { "output": { "dtype": "float32", "shape": [8, 128, 768], "dist": "empty" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "gflops", |
| "name": "projection+attention", |
| "value": "2 * args.batch * args.seq * args.inHidden * 3 * args.hidden + 4 * args.batch * args.hidden * args.seq * (args.past + args.seq)" |
| }, |
| { |
| "type": "bandwidth", |
| "name": "operands", |
| "value": "args.inHidden * 3 * args.hidden * 4 + args.batch * args.seq * args.inHidden * 4 + args.batch * args.seq * args.hidden * 4" |
| } |
| ], |
| "primary": true |
| }, |
| "provenance": { "model": "BERT-base encoder: hidden 768, 12 heads, head size 64" } |
| }, |
| { |
| "name": "qattention-bert-b8-s128-u8u8", |
| "preset": "smoke", |
| "vars": { "batch": 8, "seq": 128, "inHidden": 768, "hidden": 768, "heads": 12, "past": 0 }, |
| "attrs": { "num_heads": 12 }, |
| "inputs": { |
| "input": { "dtype": "uint8", "shape": [8, 128, 768], "dist": "randint", "min": 0, "max": 256, "seed": 101 }, |
| "weight": { "dtype": "uint8", "shape": [768, 2304], "dist": "randint", "min": 0, "max": 256, "seed": 102 }, |
| "bias": { "dtype": "float32", "shape": [2304], "dist": "normal", "scale": 0.3, "seed": 103 }, |
| "input_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.02 }, |
| "weight_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.012 }, |
| "mask_index": { "dtype": "int32", "shape": [8], "dist": "randint", "min": 1, "max": 129, "seed": 105 }, |
| "input_zero_point": { "dtype": "uint8", "shape": [1], "dist": "constant", "value": 128 }, |
| "weight_zero_point": { "dtype": "uint8", "shape": [1], "dist": "constant", "value": 129 } |
| }, |
| "outputs": { "output": { "dtype": "float32", "shape": [8, 128, 768], "dist": "empty" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "gflops", |
| "name": "projection+attention", |
| "value": "2 * args.batch * args.seq * args.inHidden * 3 * args.hidden + 4 * args.batch * args.hidden * args.seq * (args.past + args.seq)" |
| }, |
| { |
| "type": "bandwidth", |
| "name": "operands", |
| "value": "args.inHidden * 3 * args.hidden * 4 + args.batch * args.seq * args.inHidden * 4 + args.batch * args.seq * args.hidden * 4" |
| } |
| ] |
| }, |
| "provenance": { "model": "BERT-base encoder: hidden 768, 12 heads, head size 64" } |
| }, |
| { |
| "name": "qattention-bert-b1-s128-percolumn", |
| "preset": "smoke", |
| "vars": { "batch": 1, "seq": 128, "inHidden": 768, "hidden": 768, "heads": 12, "past": 0 }, |
| "attrs": { "num_heads": 12 }, |
| "inputs": { |
| "input": { "dtype": "uint8", "shape": [1, 128, 768], "dist": "randint", "min": 0, "max": 256, "seed": 101 }, |
| "weight": { "dtype": "int8", "shape": [768, 2304], "dist": "randint", "min": -128, "max": 128, "seed": 102 }, |
| "bias": { "dtype": "float32", "shape": [2304], "dist": "normal", "scale": 0.3, "seed": 103 }, |
| "input_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.02 }, |
| "weight_scale": { |
| "dtype": "float32", |
| "shape": [2304], |
| "dist": "uniform", |
| "min": 0.005, |
| "max": 0.02, |
| "seed": 104 |
| }, |
| "mask_index": { "dtype": "int32", "shape": [1], "dist": "randint", "min": 1, "max": 129, "seed": 105 }, |
| "input_zero_point": { "dtype": "uint8", "shape": [1], "dist": "constant", "value": 128 }, |
| "weight_zero_point": { "dtype": "int8", "shape": [2304], "dist": "randint", "min": -8, "max": 9, "seed": 106 } |
| }, |
| "outputs": { "output": { "dtype": "float32", "shape": [1, 128, 768], "dist": "empty" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "gflops", |
| "name": "projection+attention", |
| "value": "2 * args.batch * args.seq * args.inHidden * 3 * args.hidden + 4 * args.batch * args.hidden * args.seq * (args.past + args.seq)" |
| }, |
| { |
| "type": "bandwidth", |
| "name": "operands", |
| "value": "args.inHidden * 3 * args.hidden * 4 + args.batch * args.seq * args.inHidden * 4 + args.batch * args.seq * args.hidden * 4" |
| } |
| ] |
| }, |
| "provenance": { "model": "BERT-base encoder: hidden 768, 12 heads, head size 64" } |
| }, |
| { |
| "name": "qattention-minilm-b32-s128", |
| "preset": "smoke", |
| "vars": { "batch": 32, "seq": 128, "inHidden": 384, "hidden": 384, "heads": 12, "past": 0 }, |
| "attrs": { "num_heads": 12 }, |
| "inputs": { |
| "input": { "dtype": "uint8", "shape": [32, 128, 384], "dist": "randint", "min": 0, "max": 256, "seed": 101 }, |
| "weight": { "dtype": "int8", "shape": [384, 1152], "dist": "randint", "min": -128, "max": 128, "seed": 102 }, |
| "bias": { "dtype": "float32", "shape": [1152], "dist": "normal", "scale": 0.3, "seed": 103 }, |
| "input_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.02 }, |
| "weight_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.012 }, |
| "mask_index": { "dtype": "int32", "shape": [32], "dist": "randint", "min": 1, "max": 129, "seed": 105 }, |
| "input_zero_point": { "dtype": "uint8", "shape": [1], "dist": "constant", "value": 128 }, |
| "weight_zero_point": { "dtype": "int8", "shape": [1], "dist": "constant", "value": 1 } |
| }, |
| "outputs": { "output": { "dtype": "float32", "shape": [32, 128, 384], "dist": "empty" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "gflops", |
| "name": "projection+attention", |
| "value": "2 * args.batch * args.seq * args.inHidden * 3 * args.hidden + 4 * args.batch * args.hidden * args.seq * (args.past + args.seq)" |
| }, |
| { |
| "type": "bandwidth", |
| "name": "operands", |
| "value": "args.inHidden * 3 * args.hidden * 4 + args.batch * args.seq * args.inHidden * 4 + args.batch * args.seq * args.hidden * 4" |
| } |
| ] |
| }, |
| "provenance": { "model": "all-MiniLM-L6-v2 encoder: hidden 384, 12 heads, head size 32" } |
| }, |
| { |
| "name": "qattention-bert-decode-p127", |
| "preset": "smoke", |
| "vars": { "batch": 1, "seq": 1, "inHidden": 768, "hidden": 768, "heads": 12, "past": 127 }, |
| "attrs": { "num_heads": 12 }, |
| "inputs": { |
| "input": { "dtype": "uint8", "shape": [1, 1, 768], "dist": "randint", "min": 0, "max": 256, "seed": 101 }, |
| "weight": { "dtype": "int8", "shape": [768, 2304], "dist": "randint", "min": -128, "max": 128, "seed": 102 }, |
| "bias": { "dtype": "float32", "shape": [2304], "dist": "normal", "scale": 0.3, "seed": 103 }, |
| "input_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.02 }, |
| "weight_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.012 }, |
| "mask_index": { "dtype": "int32", "shape": [1], "dist": "randint", "min": 1, "max": 129, "seed": 105 }, |
| "input_zero_point": { "dtype": "uint8", "shape": [1], "dist": "constant", "value": 128 }, |
| "weight_zero_point": { "dtype": "int8", "shape": [1], "dist": "constant", "value": 1 }, |
| "past": { "dtype": "float32", "shape": [2, 1, 12, 127, 64], "dist": "normal", "scale": 0.3, "seed": 107 } |
| }, |
| "outputs": { |
| "output": { "dtype": "float32", "shape": [1, 1, 768], "dist": "empty" }, |
| "present": { "dtype": "float32", "shape": [2, 1, 12, 128, 64], "dist": "empty" } |
| }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "bandwidth", |
| "name": "operands", |
| "value": "args.inHidden * 3 * args.hidden * 4 + args.batch * args.seq * args.inHidden * 4 + args.batch * args.seq * args.hidden * 4 + 2 * args.batch * args.hidden * args.past * 4 + 2 * args.batch * args.hidden * (args.past + args.seq) * 4" |
| }, |
| { |
| "type": "gflops", |
| "name": "projection+attention", |
| "value": "2 * args.batch * args.seq * args.inHidden * 3 * args.hidden + 4 * args.batch * args.hidden * args.seq * (args.past + args.seq)" |
| } |
| ] |
| }, |
| "provenance": { "model": "BERT-base encoder: hidden 768, 12 heads, head size 64" } |
| }, |
| { |
| "name": "qattention-bert-b16-s384", |
| "preset": "scale", |
| "vars": { "batch": 16, "seq": 384, "inHidden": 768, "hidden": 768, "heads": 12, "past": 0 }, |
| "attrs": { "num_heads": 12 }, |
| "inputs": { |
| "input": { "dtype": "uint8", "shape": [16, 384, 768], "dist": "randint", "min": 0, "max": 256, "seed": 101 }, |
| "weight": { "dtype": "int8", "shape": [768, 2304], "dist": "randint", "min": -128, "max": 128, "seed": 102 }, |
| "bias": { "dtype": "float32", "shape": [2304], "dist": "normal", "scale": 0.3, "seed": 103 }, |
| "input_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.02 }, |
| "weight_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.012 }, |
| "mask_index": { "dtype": "int32", "shape": [16], "dist": "randint", "min": 1, "max": 385, "seed": 105 }, |
| "input_zero_point": { "dtype": "uint8", "shape": [1], "dist": "constant", "value": 128 }, |
| "weight_zero_point": { "dtype": "int8", "shape": [1], "dist": "constant", "value": 1 } |
| }, |
| "outputs": { "output": { "dtype": "float32", "shape": [16, 384, 768], "dist": "empty" } }, |
| "bench": { |
| "metrics": [ |
| { |
| "type": "gflops", |
| "name": "projection+attention", |
| "value": "2 * args.batch * args.seq * args.inHidden * 3 * args.hidden + 4 * args.batch * args.hidden * args.seq * (args.past + args.seq)" |
| }, |
| { |
| "type": "bandwidth", |
| "name": "operands", |
| "value": "args.inHidden * 3 * args.hidden * 4 + args.batch * args.seq * args.inHidden * 4 + args.batch * args.seq * args.hidden * 4" |
| } |
| ] |
| }, |
| "provenance": { "model": "BERT-base encoder: hidden 768, 12 heads, head size 64" } |
| } |
| ] |
| } |
|
|
