File size: 13,307 Bytes
6454177 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 | {
"tunableSpace": {
"DP4A_TILE_M": [8, 16, 32, 64],
"DP4A_TILE_N": [32, 64, 128],
"DP4A_K_TILE_WORDS": [8, 16, 32],
"ATTEND_WORKGROUP": [32, 64, 128, 256]
},
"cases": [
{
"name": "qattention-bert-b1-s128",
"preset": "smoke",
"vars": { "batch": 1, "seq": 128, "inHidden": 768, "hidden": 768, "heads": 12, "past": 0 },
"attrs": { "num_heads": 12 },
"inputs": {
"input": { "dtype": "uint8", "shape": [1, 128, 768], "dist": "randint", "min": 0, "max": 256, "seed": 101 },
"weight": { "dtype": "int8", "shape": [768, 2304], "dist": "randint", "min": -128, "max": 128, "seed": 102 },
"bias": { "dtype": "float32", "shape": [2304], "dist": "normal", "scale": 0.3, "seed": 103 },
"input_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.02 },
"weight_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.012 },
"mask_index": { "dtype": "int32", "shape": [1], "dist": "randint", "min": 1, "max": 129, "seed": 105 },
"input_zero_point": { "dtype": "uint8", "shape": [1], "dist": "constant", "value": 128 },
"weight_zero_point": { "dtype": "int8", "shape": [1], "dist": "constant", "value": 1 }
},
"outputs": { "output": { "dtype": "float32", "shape": [1, 128, 768], "dist": "empty" } },
"bench": {
"metrics": [
{
"type": "gflops",
"name": "projection+attention",
"value": "2 * args.batch * args.seq * args.inHidden * 3 * args.hidden + 4 * args.batch * args.hidden * args.seq * (args.past + args.seq)"
},
{
"type": "bandwidth",
"name": "operands",
"value": "args.inHidden * 3 * args.hidden * 4 + args.batch * args.seq * args.inHidden * 4 + args.batch * args.seq * args.hidden * 4"
}
]
},
"provenance": { "model": "BERT-base encoder: hidden 768, 12 heads, head size 64" }
},
{
"name": "qattention-bert-b8-s128",
"preset": "smoke",
"vars": { "batch": 8, "seq": 128, "inHidden": 768, "hidden": 768, "heads": 12, "past": 0 },
"attrs": { "num_heads": 12 },
"inputs": {
"input": { "dtype": "uint8", "shape": [8, 128, 768], "dist": "randint", "min": 0, "max": 256, "seed": 101 },
"weight": { "dtype": "int8", "shape": [768, 2304], "dist": "randint", "min": -128, "max": 128, "seed": 102 },
"bias": { "dtype": "float32", "shape": [2304], "dist": "normal", "scale": 0.3, "seed": 103 },
"input_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.02 },
"weight_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.012 },
"mask_index": { "dtype": "int32", "shape": [8], "dist": "randint", "min": 1, "max": 129, "seed": 105 },
"input_zero_point": { "dtype": "uint8", "shape": [1], "dist": "constant", "value": 128 },
"weight_zero_point": { "dtype": "int8", "shape": [1], "dist": "constant", "value": 1 }
},
"outputs": { "output": { "dtype": "float32", "shape": [8, 128, 768], "dist": "empty" } },
"bench": {
"metrics": [
{
"type": "gflops",
"name": "projection+attention",
"value": "2 * args.batch * args.seq * args.inHidden * 3 * args.hidden + 4 * args.batch * args.hidden * args.seq * (args.past + args.seq)"
},
{
"type": "bandwidth",
"name": "operands",
"value": "args.inHidden * 3 * args.hidden * 4 + args.batch * args.seq * args.inHidden * 4 + args.batch * args.seq * args.hidden * 4"
}
],
"primary": true
},
"provenance": { "model": "BERT-base encoder: hidden 768, 12 heads, head size 64" }
},
{
"name": "qattention-bert-b8-s128-u8u8",
"preset": "smoke",
"vars": { "batch": 8, "seq": 128, "inHidden": 768, "hidden": 768, "heads": 12, "past": 0 },
"attrs": { "num_heads": 12 },
"inputs": {
"input": { "dtype": "uint8", "shape": [8, 128, 768], "dist": "randint", "min": 0, "max": 256, "seed": 101 },
"weight": { "dtype": "uint8", "shape": [768, 2304], "dist": "randint", "min": 0, "max": 256, "seed": 102 },
"bias": { "dtype": "float32", "shape": [2304], "dist": "normal", "scale": 0.3, "seed": 103 },
"input_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.02 },
"weight_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.012 },
"mask_index": { "dtype": "int32", "shape": [8], "dist": "randint", "min": 1, "max": 129, "seed": 105 },
"input_zero_point": { "dtype": "uint8", "shape": [1], "dist": "constant", "value": 128 },
"weight_zero_point": { "dtype": "uint8", "shape": [1], "dist": "constant", "value": 129 }
},
"outputs": { "output": { "dtype": "float32", "shape": [8, 128, 768], "dist": "empty" } },
"bench": {
"metrics": [
{
"type": "gflops",
"name": "projection+attention",
"value": "2 * args.batch * args.seq * args.inHidden * 3 * args.hidden + 4 * args.batch * args.hidden * args.seq * (args.past + args.seq)"
},
{
"type": "bandwidth",
"name": "operands",
"value": "args.inHidden * 3 * args.hidden * 4 + args.batch * args.seq * args.inHidden * 4 + args.batch * args.seq * args.hidden * 4"
}
]
},
"provenance": { "model": "BERT-base encoder: hidden 768, 12 heads, head size 64" }
},
{
"name": "qattention-bert-b1-s128-percolumn",
"preset": "smoke",
"vars": { "batch": 1, "seq": 128, "inHidden": 768, "hidden": 768, "heads": 12, "past": 0 },
"attrs": { "num_heads": 12 },
"inputs": {
"input": { "dtype": "uint8", "shape": [1, 128, 768], "dist": "randint", "min": 0, "max": 256, "seed": 101 },
"weight": { "dtype": "int8", "shape": [768, 2304], "dist": "randint", "min": -128, "max": 128, "seed": 102 },
"bias": { "dtype": "float32", "shape": [2304], "dist": "normal", "scale": 0.3, "seed": 103 },
"input_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.02 },
"weight_scale": {
"dtype": "float32",
"shape": [2304],
"dist": "uniform",
"min": 0.005,
"max": 0.02,
"seed": 104
},
"mask_index": { "dtype": "int32", "shape": [1], "dist": "randint", "min": 1, "max": 129, "seed": 105 },
"input_zero_point": { "dtype": "uint8", "shape": [1], "dist": "constant", "value": 128 },
"weight_zero_point": { "dtype": "int8", "shape": [2304], "dist": "randint", "min": -8, "max": 9, "seed": 106 }
},
"outputs": { "output": { "dtype": "float32", "shape": [1, 128, 768], "dist": "empty" } },
"bench": {
"metrics": [
{
"type": "gflops",
"name": "projection+attention",
"value": "2 * args.batch * args.seq * args.inHidden * 3 * args.hidden + 4 * args.batch * args.hidden * args.seq * (args.past + args.seq)"
},
{
"type": "bandwidth",
"name": "operands",
"value": "args.inHidden * 3 * args.hidden * 4 + args.batch * args.seq * args.inHidden * 4 + args.batch * args.seq * args.hidden * 4"
}
]
},
"provenance": { "model": "BERT-base encoder: hidden 768, 12 heads, head size 64" }
},
{
"name": "qattention-minilm-b32-s128",
"preset": "smoke",
"vars": { "batch": 32, "seq": 128, "inHidden": 384, "hidden": 384, "heads": 12, "past": 0 },
"attrs": { "num_heads": 12 },
"inputs": {
"input": { "dtype": "uint8", "shape": [32, 128, 384], "dist": "randint", "min": 0, "max": 256, "seed": 101 },
"weight": { "dtype": "int8", "shape": [384, 1152], "dist": "randint", "min": -128, "max": 128, "seed": 102 },
"bias": { "dtype": "float32", "shape": [1152], "dist": "normal", "scale": 0.3, "seed": 103 },
"input_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.02 },
"weight_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.012 },
"mask_index": { "dtype": "int32", "shape": [32], "dist": "randint", "min": 1, "max": 129, "seed": 105 },
"input_zero_point": { "dtype": "uint8", "shape": [1], "dist": "constant", "value": 128 },
"weight_zero_point": { "dtype": "int8", "shape": [1], "dist": "constant", "value": 1 }
},
"outputs": { "output": { "dtype": "float32", "shape": [32, 128, 384], "dist": "empty" } },
"bench": {
"metrics": [
{
"type": "gflops",
"name": "projection+attention",
"value": "2 * args.batch * args.seq * args.inHidden * 3 * args.hidden + 4 * args.batch * args.hidden * args.seq * (args.past + args.seq)"
},
{
"type": "bandwidth",
"name": "operands",
"value": "args.inHidden * 3 * args.hidden * 4 + args.batch * args.seq * args.inHidden * 4 + args.batch * args.seq * args.hidden * 4"
}
]
},
"provenance": { "model": "all-MiniLM-L6-v2 encoder: hidden 384, 12 heads, head size 32" }
},
{
"name": "qattention-bert-decode-p127",
"preset": "smoke",
"vars": { "batch": 1, "seq": 1, "inHidden": 768, "hidden": 768, "heads": 12, "past": 127 },
"attrs": { "num_heads": 12 },
"inputs": {
"input": { "dtype": "uint8", "shape": [1, 1, 768], "dist": "randint", "min": 0, "max": 256, "seed": 101 },
"weight": { "dtype": "int8", "shape": [768, 2304], "dist": "randint", "min": -128, "max": 128, "seed": 102 },
"bias": { "dtype": "float32", "shape": [2304], "dist": "normal", "scale": 0.3, "seed": 103 },
"input_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.02 },
"weight_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.012 },
"mask_index": { "dtype": "int32", "shape": [1], "dist": "randint", "min": 1, "max": 129, "seed": 105 },
"input_zero_point": { "dtype": "uint8", "shape": [1], "dist": "constant", "value": 128 },
"weight_zero_point": { "dtype": "int8", "shape": [1], "dist": "constant", "value": 1 },
"past": { "dtype": "float32", "shape": [2, 1, 12, 127, 64], "dist": "normal", "scale": 0.3, "seed": 107 }
},
"outputs": {
"output": { "dtype": "float32", "shape": [1, 1, 768], "dist": "empty" },
"present": { "dtype": "float32", "shape": [2, 1, 12, 128, 64], "dist": "empty" }
},
"bench": {
"metrics": [
{
"type": "bandwidth",
"name": "operands",
"value": "args.inHidden * 3 * args.hidden * 4 + args.batch * args.seq * args.inHidden * 4 + args.batch * args.seq * args.hidden * 4 + 2 * args.batch * args.hidden * args.past * 4 + 2 * args.batch * args.hidden * (args.past + args.seq) * 4"
},
{
"type": "gflops",
"name": "projection+attention",
"value": "2 * args.batch * args.seq * args.inHidden * 3 * args.hidden + 4 * args.batch * args.hidden * args.seq * (args.past + args.seq)"
}
]
},
"provenance": { "model": "BERT-base encoder: hidden 768, 12 heads, head size 64" }
},
{
"name": "qattention-bert-b16-s384",
"preset": "scale",
"vars": { "batch": 16, "seq": 384, "inHidden": 768, "hidden": 768, "heads": 12, "past": 0 },
"attrs": { "num_heads": 12 },
"inputs": {
"input": { "dtype": "uint8", "shape": [16, 384, 768], "dist": "randint", "min": 0, "max": 256, "seed": 101 },
"weight": { "dtype": "int8", "shape": [768, 2304], "dist": "randint", "min": -128, "max": 128, "seed": 102 },
"bias": { "dtype": "float32", "shape": [2304], "dist": "normal", "scale": 0.3, "seed": 103 },
"input_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.02 },
"weight_scale": { "dtype": "float32", "shape": [1], "dist": "constant", "value": 0.012 },
"mask_index": { "dtype": "int32", "shape": [16], "dist": "randint", "min": 1, "max": 385, "seed": 105 },
"input_zero_point": { "dtype": "uint8", "shape": [1], "dist": "constant", "value": 128 },
"weight_zero_point": { "dtype": "int8", "shape": [1], "dist": "constant", "value": 1 }
},
"outputs": { "output": { "dtype": "float32", "shape": [16, 384, 768], "dist": "empty" } },
"bench": {
"metrics": [
{
"type": "gflops",
"name": "projection+attention",
"value": "2 * args.batch * args.seq * args.inHidden * 3 * args.hidden + 4 * args.batch * args.hidden * args.seq * (args.past + args.seq)"
},
{
"type": "bandwidth",
"name": "operands",
"value": "args.inHidden * 3 * args.hidden * 4 + args.batch * args.seq * args.inHidden * 4 + args.batch * args.seq * args.hidden * 4"
}
]
},
"provenance": { "model": "BERT-base encoder: hidden 768, 12 heads, head size 64" }
}
]
}
|