File size: 32,840 Bytes
2443b58 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 | {
"op": "com.microsoft.MultiHeadAttention",
"cases": [
{
"name": "minilm-self-attention-b1-s20-h12-d32",
"provenance": {
"notes": "Upper boundary of the whole-head small-sequence route at 12 heads and sequence length 20. Paired with the sequence-24 case, it checks that the mutually exclusive route predicates meet without a gap."
},
"vars": { "batch": 1, "qSeq": 20, "kvSeq": 20, "heads": 12, "headDim": 32 },
"attrs": { "num_heads": 12 },
"inputs": {
"queryT": { "shape": [1, 20, 384], "dtype": "float32", "dist": "normal", "seed": 9251, "scale": 0.2 },
"keyT": { "shape": [1, 20, 384], "dtype": "float32", "dist": "normal", "seed": 9252, "scale": 0.2 },
"valueT": { "shape": [1, 20, 384], "dtype": "float32", "dist": "normal", "seed": 9253, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 20, 384], "dtype": "float32" } },
"bench": {
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" }
]
}
},
{
"name": "minilm-self-attention-b1-s24-h12-d32",
"provenance": {
"notes": "Boundary immediately above the whole-head small-sequence route. This shape satisfies flashShapeOk, so it checks that the mutually exclusive route predicates remain aligned."
},
"vars": { "batch": 1, "qSeq": 24, "kvSeq": 24, "heads": 12, "headDim": 32 },
"attrs": { "num_heads": 12 },
"inputs": {
"queryT": { "shape": [1, 24, 384], "dtype": "float32", "dist": "normal", "seed": 9261, "scale": 0.2 },
"keyT": { "shape": [1, 24, 384], "dtype": "float32", "dist": "normal", "seed": 9262, "scale": 0.2 },
"valueT": { "shape": [1, 24, 384], "dtype": "float32", "dist": "normal", "seed": 9263, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 24, 384], "dtype": "float32" } },
"bench": {
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" }
]
}
},
{
"name": "minilm-self-attention-b1-s11-h12-d32",
"provenance": {
"notes": "all-MiniLM-L6-v2 self-attention at a short sentence: 12 heads of 32, bidirectional, no bias."
},
"vars": { "batch": 1, "qSeq": 11, "kvSeq": 11, "heads": 12, "headDim": 32 },
"attrs": { "num_heads": 12 },
"inputs": {
"queryT": { "shape": [1, 11, 384], "dtype": "float32", "dist": "normal", "seed": 9201, "scale": 0.2 },
"keyT": { "shape": [1, 11, 384], "dtype": "float32", "dist": "normal", "seed": 9202, "scale": 0.2 },
"valueT": { "shape": [1, 11, 384], "dtype": "float32", "dist": "normal", "seed": 9203, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 11, 384], "dtype": "float32" } },
"bench": {
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" }
]
}
},
{
"name": "minilm-self-attention-b1-s32-h12-d32",
"provenance": { "notes": "all-MiniLM-L6-v2 self-attention: 12 heads of 32, bidirectional, no bias." },
"vars": { "batch": 1, "qSeq": 32, "kvSeq": 32, "heads": 12, "headDim": 32 },
"attrs": { "num_heads": 12 },
"inputs": {
"queryT": { "shape": [1, 32, 384], "dtype": "float32", "dist": "normal", "seed": 9211, "scale": 0.2 },
"keyT": { "shape": [1, 32, 384], "dtype": "float32", "dist": "normal", "seed": 9212, "scale": 0.2 },
"valueT": { "shape": [1, 32, 384], "dtype": "float32", "dist": "normal", "seed": 9213, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 32, 384], "dtype": "float32" } },
"bench": {
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" }
]
}
},
{
"name": "minilm-self-attention-b1-s128-h12-d32",
"provenance": { "notes": "all-MiniLM-L6-v2 self-attention: 12 heads of 32, bidirectional, no bias." },
"vars": { "batch": 1, "qSeq": 128, "kvSeq": 128, "heads": 12, "headDim": 32 },
"attrs": { "num_heads": 12 },
"inputs": {
"queryT": { "shape": [1, 128, 384], "dtype": "float32", "dist": "normal", "seed": 9221, "scale": 0.2 },
"keyT": { "shape": [1, 128, 384], "dtype": "float32", "dist": "normal", "seed": 9222, "scale": 0.2 },
"valueT": { "shape": [1, 128, 384], "dtype": "float32", "dist": "normal", "seed": 9223, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 128, 384], "dtype": "float32" } },
"bench": {
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" }
]
}
},
{
"name": "mha-f32-1x64x64-h4d16",
"preset": "smoke",
"vars": { "batch": 1, "qSeq": 64, "kvSeq": 64, "heads": 4, "headDim": 16 },
"attrs": { "num_heads": 4, "scale": 0.25 },
"inputs": {
"queryT": { "shape": [1, 64, 64], "dtype": "float32", "dist": "normal", "seed": 203, "scale": 0.2 },
"keyT": { "shape": [1, 64, 64], "dtype": "float32", "dist": "normal", "seed": 204, "scale": 0.2 },
"valueT": { "shape": [1, 64, 64], "dtype": "float32", "dist": "normal", "seed": 205, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 64, 64], "dtype": "float32" } },
"bench": {
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" }
]
}
},
{
"name": "mha-f32-bias-causal-1x64x64-h4d16",
"preset": "smoke",
"vars": { "batch": 1, "qSeq": 64, "kvSeq": 64, "heads": 4, "headDim": 16 },
"attrs": { "num_heads": 4, "scale": 0.25, "unidirectional": 1 },
"inputs": {
"queryT": { "shape": [1, 64, 64], "dtype": "float32", "dist": "normal", "seed": 204, "scale": 0.2 },
"keyT": { "shape": [1, 64, 64], "dtype": "float32", "dist": "normal", "seed": 205, "scale": 0.2 },
"valueT": { "shape": [1, 64, 64], "dtype": "float32", "dist": "normal", "seed": 206, "scale": 0.2 },
"biasT": { "shape": [192], "dtype": "float32", "dist": "normal", "seed": 207, "scale": 0.05 }
},
"outputs": { "outputT": { "shape": [1, 64, 64], "dtype": "float32" } },
"bench": {
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.heads * args.headDim * args.qSeq * (args.qSeq + 1) / 2" }
]
}
},
{
"name": "decode-h8-d64-kv512",
"preset": "smoke",
"vars": { "batch": 1, "qSeq": 1, "kvSeq": 512, "heads": 8, "headDim": 64 },
"attrs": { "num_heads": 8, "scale": 0.125 },
"inputs": {
"queryT": { "shape": [1, 1, 512], "dtype": "float32", "dist": "normal", "seed": 467, "scale": 0.2 },
"keyT": { "shape": [1, 512, 512], "dtype": "float32", "dist": "normal", "seed": 468, "scale": 0.2 },
"valueT": { "shape": [1, 512, 512], "dtype": "float32", "dist": "normal", "seed": 469, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 1, 512], "dtype": "float32" } },
"bench": {
"metrics": [
{
"type": "bandwidth",
"value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim) * 4"
}
]
}
},
{
"name": "decode-h8-d64-kv512-bias",
"preset": "smoke",
"vars": { "batch": 1, "qSeq": 1, "kvSeq": 512, "heads": 8, "headDim": 64 },
"attrs": { "num_heads": 8, "scale": 0.125 },
"inputs": {
"queryT": { "shape": [1, 1, 512], "dtype": "float32", "dist": "normal", "seed": 471, "scale": 0.2 },
"keyT": { "shape": [1, 512, 512], "dtype": "float32", "dist": "normal", "seed": 472, "scale": 0.2 },
"valueT": { "shape": [1, 512, 512], "dtype": "float32", "dist": "normal", "seed": 473, "scale": 0.2 },
"biasT": { "shape": [1536], "dtype": "float32", "dist": "normal", "seed": 474, "scale": 0.05 }
},
"outputs": { "outputT": { "shape": [1, 1, 512], "dtype": "float32" } },
"bench": {
"metrics": [
{
"type": "bandwidth",
"value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim + 3 * args.heads * args.headDim) * 4"
}
]
}
},
{
"name": "decode-h8-d64-kv768",
"preset": "smoke",
"vars": { "batch": 1, "qSeq": 1, "kvSeq": 768, "heads": 8, "headDim": 64 },
"attrs": { "num_heads": 8, "scale": 0.125 },
"inputs": {
"queryT": { "shape": [1, 1, 512], "dtype": "float32", "dist": "normal", "seed": 477, "scale": 0.2 },
"keyT": { "shape": [1, 768, 512], "dtype": "float32", "dist": "normal", "seed": 478, "scale": 0.2 },
"valueT": { "shape": [1, 768, 512], "dtype": "float32", "dist": "normal", "seed": 479, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 1, 512], "dtype": "float32" } },
"bench": {
"metrics": [
{
"type": "bandwidth",
"value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim) * 4"
}
]
}
},
{
"name": "decode-h8-d64-kv768-bias",
"preset": "smoke",
"vars": { "batch": 1, "qSeq": 1, "kvSeq": 768, "heads": 8, "headDim": 64 },
"attrs": { "num_heads": 8, "scale": 0.125 },
"inputs": {
"queryT": { "shape": [1, 1, 512], "dtype": "float32", "dist": "normal", "seed": 481, "scale": 0.2 },
"keyT": { "shape": [1, 768, 512], "dtype": "float32", "dist": "normal", "seed": 482, "scale": 0.2 },
"valueT": { "shape": [1, 768, 512], "dtype": "float32", "dist": "normal", "seed": 483, "scale": 0.2 },
"biasT": { "shape": [1536], "dtype": "float32", "dist": "normal", "seed": 484, "scale": 0.05 }
},
"outputs": { "outputT": { "shape": [1, 1, 512], "dtype": "float32" } },
"bench": {
"metrics": [
{
"type": "bandwidth",
"value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim + 3 * args.heads * args.headDim) * 4"
}
]
}
},
{
"name": "decode-h4-d64-kv512",
"preset": "smoke",
"vars": { "batch": 1, "qSeq": 1, "kvSeq": 512, "heads": 4, "headDim": 64 },
"attrs": { "num_heads": 4, "scale": 0.125 },
"inputs": {
"queryT": { "shape": [1, 1, 256], "dtype": "float32", "dist": "normal", "seed": 367, "scale": 0.2 },
"keyT": { "shape": [1, 512, 256], "dtype": "float32", "dist": "normal", "seed": 368, "scale": 0.2 },
"valueT": { "shape": [1, 512, 256], "dtype": "float32", "dist": "normal", "seed": 369, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 1, 256], "dtype": "float32" } },
"bench": {
"metrics": [
{
"type": "bandwidth",
"value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim) * 4"
}
]
}
},
{
"name": "decode-h4-d64-kv512-bias",
"preset": "smoke",
"vars": { "batch": 1, "qSeq": 1, "kvSeq": 512, "heads": 4, "headDim": 64 },
"attrs": { "num_heads": 4, "scale": 0.125 },
"inputs": {
"queryT": { "shape": [1, 1, 256], "dtype": "float32", "dist": "normal", "seed": 371, "scale": 0.2 },
"keyT": { "shape": [1, 512, 256], "dtype": "float32", "dist": "normal", "seed": 372, "scale": 0.2 },
"valueT": { "shape": [1, 512, 256], "dtype": "float32", "dist": "normal", "seed": 373, "scale": 0.2 },
"biasT": { "shape": [768], "dtype": "float32", "dist": "normal", "seed": 374, "scale": 0.05 }
},
"outputs": { "outputT": { "shape": [1, 1, 256], "dtype": "float32" } },
"bench": {
"metrics": [
{
"type": "bandwidth",
"value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim + 3 * args.heads * args.headDim) * 4"
}
]
}
},
{
"name": "decode-h8-d128-kv2048",
"preset": "model",
"vars": { "batch": 1, "qSeq": 1, "kvSeq": 2048, "heads": 8, "headDim": 128 },
"attrs": { "num_heads": 8, "scale": 0.08838834764831845 },
"inputs": {
"queryT": { "shape": [1, 1, 1024], "dtype": "float32", "dist": "normal", "seed": 1367, "scale": 0.2 },
"keyT": { "shape": [1, 2048, 1024], "dtype": "float32", "dist": "normal", "seed": 1368, "scale": 0.2 },
"valueT": { "shape": [1, 2048, 1024], "dtype": "float32", "dist": "normal", "seed": 1369, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 1, 1024], "dtype": "float32" } },
"bench": {
"primary": true,
"metrics": [
{
"type": "bandwidth",
"value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim) * 4"
}
]
}
},
{
"name": "flux2-joint-attn-img512-s1536-24h-d128",
"preset": "model",
"vars": { "batch": 1, "qSeq": 1536, "kvSeq": 1536, "heads": 24, "headDim": 128 },
"attrs": { "num_heads": 24, "scale": 0.08838834764831845 },
"inputs": {
"queryT": { "shape": [1, 1536, 3072], "dtype": "float32", "dist": "normal", "seed": 360, "scale": 0.2 },
"keyT": { "shape": [1, 1536, 3072], "dtype": "float32", "dist": "normal", "seed": 361, "scale": 0.2 },
"valueT": { "shape": [1, 1536, 3072], "dtype": "float32", "dist": "normal", "seed": 362, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 1536, 3072], "dtype": "float32" } },
"bench": {
"primary": true,
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" }
]
},
"tunableSpace": { "MATERIALIZED_FUSED_SOFTMAX_MIN_SCORE_BYTES": [16777216, 268435456] }
},
{
"name": "encoder-prefill-q1500-kv1500-h6-d64-tail",
"preset": "model",
"vars": { "batch": 1, "qSeq": 1500, "kvSeq": 1500, "heads": 6, "headDim": 64 },
"attrs": { "num_heads": 6, "scale": 0.125 },
"tunableSpace": {
"MATERIALIZED_SGMAT_QUERY_TILE": [16, 32, 64],
"MATERIALIZED_SGMAT_KEY_TILE": [32, 64],
"MATERIALIZED_FUSED_SOFTMAX_MIN_SCORE_BYTES": [16777216, 268435456]
},
"inputs": {
"queryT": { "shape": [1, 1500, 384], "dtype": "float32", "dist": "normal", "seed": 460, "scale": 0.2 },
"keyT": { "shape": [1, 1500, 384], "dtype": "float32", "dist": "normal", "seed": 461, "scale": 0.2 },
"valueT": { "shape": [1, 1500, 384], "dtype": "float32", "dist": "normal", "seed": 462, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 1500, 384], "dtype": "float32" } },
"bench": {
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" }
]
}
},
{
"name": "decode-cross-q1-kv1500-h6-d64-underfilled-wide-subgroup",
"preset": "model",
"vars": { "batch": 1, "qSeq": 1, "kvSeq": 1500, "heads": 6, "headDim": 64 },
"attrs": { "num_heads": 6 },
"tunableSpace": {
"WORKGROUP_SIZE": [64, 128, 256],
"DECODE_MAX_SPLITS": [4, 6, 8, 10, 12, 16],
"DECODE_KEYS_PER_SPLIT": [128]
},
"inputs": {
"queryT": { "shape": [1, 1, 384], "dtype": "float32", "dist": "normal", "seed": 463, "scale": 0.2 },
"keyT": { "shape": [1, 1500, 384], "dtype": "float32", "dist": "normal", "seed": 464, "scale": 0.2 },
"valueT": { "shape": [1, 1500, 384], "dtype": "float32", "dist": "normal", "seed": 465, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 1, 384], "dtype": "float32" } },
"bench": {
"metrics": [
{
"type": "bandwidth",
"value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim) * 4"
}
]
}
},
{
"name": "flux2-joint-attn-img512-s1536-24h-d128-bias",
"preset": "model",
"vars": { "batch": 1, "qSeq": 1536, "kvSeq": 1536, "heads": 24, "headDim": 128 },
"attrs": { "num_heads": 24, "scale": 0.08838834764831845 },
"inputs": {
"queryT": { "shape": [1, 1536, 3072], "dtype": "float32", "dist": "normal", "seed": 363, "scale": 0.2 },
"keyT": { "shape": [1, 1536, 3072], "dtype": "float32", "dist": "normal", "seed": 364, "scale": 0.2 },
"valueT": { "shape": [1, 1536, 3072], "dtype": "float32", "dist": "normal", "seed": 365, "scale": 0.2 },
"biasT": { "shape": [9216], "dtype": "float32", "dist": "normal", "seed": 366, "scale": 0.05 }
},
"outputs": { "outputT": { "shape": [1, 1536, 3072], "dtype": "float32" } },
"bench": {
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" }
]
}
},
{
"name": "flux2-joint-attn-img512-s1536-24h-d128-f16",
"preset": "model",
"vars": { "batch": 1, "qSeq": 1536, "kvSeq": 1536, "heads": 24, "headDim": 128 },
"attrs": { "num_heads": 24, "scale": 0.08838834764831845 },
"inputs": {
"queryT": { "shape": [1, 1536, 3072], "dtype": "float16", "dist": "normal", "seed": 360, "scale": 0.2 },
"keyT": { "shape": [1, 1536, 3072], "dtype": "float16", "dist": "normal", "seed": 361, "scale": 0.2 },
"valueT": { "shape": [1, 1536, 3072], "dtype": "float16", "dist": "normal", "seed": 362, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 1536, 3072], "dtype": "float16" } },
"bench": {
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" }
]
}
},
{
"name": "decode-h8-d128-kv2048-f16",
"preset": "model",
"vars": { "batch": 1, "qSeq": 1, "kvSeq": 2048, "heads": 8, "headDim": 128 },
"attrs": { "num_heads": 8, "scale": 0.08838834764831845 },
"inputs": {
"queryT": { "shape": [1, 1, 1024], "dtype": "float16", "dist": "normal", "seed": 1367, "scale": 0.2 },
"keyT": { "shape": [1, 2048, 1024], "dtype": "float16", "dist": "normal", "seed": 1368, "scale": 0.2 },
"valueT": { "shape": [1, 2048, 1024], "dtype": "float16", "dist": "normal", "seed": 1369, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 1, 1024], "dtype": "float16" } },
"bench": {
"metrics": [
{
"type": "bandwidth",
"value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim) * 2"
}
]
}
},
{
"name": "prefill-lpq4-d80-s512-h8-BLINDSPOT",
"preset": "smoke",
"vars": { "batch": 1, "qSeq": 512, "kvSeq": 512, "heads": 8, "headDim": 80 },
"attrs": { "num_heads": 8, "scale": 0.111803398875 },
"inputs": {
"queryT": { "shape": [1, 512, 640], "dtype": "float32", "dist": "normal", "seed": 5101, "scale": 0.2 },
"keyT": { "shape": [1, 512, 640], "dtype": "float32", "dist": "normal", "seed": 5102, "scale": 0.2 },
"valueT": { "shape": [1, 512, 640], "dtype": "float32", "dist": "normal", "seed": 5103, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 512, 640], "dtype": "float32" } },
"bench": {
"primary": true,
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" }
]
}
},
{
"name": "prefill-cluster-d128-s512-h8-HEALTHY",
"preset": "smoke",
"vars": { "batch": 1, "qSeq": 512, "kvSeq": 512, "heads": 8, "headDim": 128 },
"attrs": { "num_heads": 8, "scale": 0.08838834764831845 },
"inputs": {
"queryT": { "shape": [1, 512, 1024], "dtype": "float32", "dist": "normal", "seed": 5111, "scale": 0.2 },
"keyT": { "shape": [1, 512, 1024], "dtype": "float32", "dist": "normal", "seed": 5112, "scale": 0.2 },
"valueT": { "shape": [1, 512, 1024], "dtype": "float32", "dist": "normal", "seed": 5113, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 512, 1024], "dtype": "float32" } },
"bench": {
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" }
]
}
},
{
"name": "prefill-online-noncluster-qseq16-h16-d64-FALLOFF",
"preset": "smoke",
"vars": { "batch": 1, "qSeq": 16, "kvSeq": 512, "heads": 16, "headDim": 64 },
"attrs": { "num_heads": 16, "scale": 0.125 },
"inputs": {
"queryT": { "shape": [1, 16, 1024], "dtype": "float32", "dist": "normal", "seed": 5121, "scale": 0.2 },
"keyT": { "shape": [1, 512, 1024], "dtype": "float32", "dist": "normal", "seed": 5122, "scale": 0.2 },
"valueT": { "shape": [1, 512, 1024], "dtype": "float32", "dist": "normal", "seed": 5123, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 16, 1024], "dtype": "float32" } },
"bench": {
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" }
]
}
},
{
"name": "prefill-cluster-qseq64-h16-d64-HEALTHY",
"preset": "smoke",
"vars": { "batch": 1, "qSeq": 64, "kvSeq": 512, "heads": 16, "headDim": 64 },
"attrs": { "num_heads": 16, "scale": 0.125 },
"inputs": {
"queryT": { "shape": [1, 64, 1024], "dtype": "float32", "dist": "normal", "seed": 5131, "scale": 0.2 },
"keyT": { "shape": [1, 512, 1024], "dtype": "float32", "dist": "normal", "seed": 5132, "scale": 0.2 },
"valueT": { "shape": [1, 512, 1024], "dtype": "float32", "dist": "normal", "seed": 5133, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 64, 1024], "dtype": "float32" } },
"bench": {
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" }
]
}
},
{
"name": "decode-splitk-longkv-kv16384-h4-d64-LONGCONTEXT",
"preset": "model",
"vars": { "batch": 1, "qSeq": 1, "kvSeq": 16384, "heads": 4, "headDim": 64 },
"attrs": { "num_heads": 4, "scale": 0.125 },
"inputs": {
"queryT": { "shape": [1, 1, 256], "dtype": "float32", "dist": "normal", "seed": 5141, "scale": 0.2 },
"keyT": { "shape": [1, 16384, 256], "dtype": "float32", "dist": "normal", "seed": 5142, "scale": 0.2 },
"valueT": { "shape": [1, 16384, 256], "dtype": "float32", "dist": "normal", "seed": 5143, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 1, 256], "dtype": "float32" } },
"bench": {
"metrics": [
{
"type": "bandwidth",
"value": "(2 * args.kvSeq * args.heads * args.headDim + 2 * args.qSeq * args.heads * args.headDim) * 4"
}
]
}
},
{
"name": "scalar-fallback-odd-headdim6-prefill-h8-q128-kv2048-SLOW",
"preset": "stress",
"vars": { "batch": 1, "qSeq": 128, "kvSeq": 2048, "heads": 8, "headDim": 6 },
"attrs": { "num_heads": 8, "scale": 0.408248290463863 },
"inputs": {
"queryT": { "shape": [1, 128, 48], "dtype": "float32", "dist": "normal", "seed": 6201, "scale": 0.2 },
"keyT": { "shape": [1, 2048, 48], "dtype": "float32", "dist": "normal", "seed": 6202, "scale": 0.2 },
"valueT": { "shape": [1, 2048, 48], "dtype": "float32", "dist": "normal", "seed": 6203, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 128, 48], "dtype": "float32" } },
"bench": {
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" }
]
}
},
{
"name": "scalar-fallback-shortq16-longkv4096-d128-h4-b2-SLOW",
"preset": "stress",
"vars": { "batch": 2, "qSeq": 16, "kvSeq": 4096, "heads": 4, "headDim": 128 },
"attrs": { "num_heads": 4, "scale": 0.08838834764831845 },
"inputs": {
"queryT": { "shape": [2, 16, 512], "dtype": "float32", "dist": "normal", "seed": 6211, "scale": 0.2 },
"keyT": { "shape": [2, 4096, 512], "dtype": "float32", "dist": "normal", "seed": 6212, "scale": 0.2 },
"valueT": { "shape": [2, 4096, 512], "dtype": "float32", "dist": "normal", "seed": 6213, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [2, 16, 512], "dtype": "float32" } },
"bench": {
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" }
]
}
},
{
"name": "mha-f32-q31-kv511-h8-d64-double-threshold-cliff",
"preset": "stress",
"vars": { "batch": 1, "qSeq": 31, "kvSeq": 511, "heads": 8, "headDim": 64 },
"attrs": { "num_heads": 8, "scale": 0.125 },
"inputs": {
"queryT": { "shape": [1, 31, 512], "dtype": "float32", "dist": "normal", "seed": 6221, "scale": 0.2 },
"keyT": { "shape": [1, 511, 512], "dtype": "float32", "dist": "normal", "seed": 6222, "scale": 0.2 },
"valueT": { "shape": [1, 511, 512], "dtype": "float32", "dist": "normal", "seed": 6223, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 31, 512], "dtype": "float32", "dist": "empty" } },
"bench": {
"primary": true,
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" }
]
}
},
{
"name": "mha-f32-q32-kv512-h8-d64-threshold-control",
"preset": "stress",
"vars": { "batch": 1, "qSeq": 32, "kvSeq": 512, "heads": 8, "headDim": 64 },
"attrs": { "num_heads": 8, "scale": 0.125 },
"inputs": {
"queryT": { "shape": [1, 32, 512], "dtype": "float32", "dist": "normal", "seed": 6224, "scale": 0.2 },
"keyT": { "shape": [1, 512, 512], "dtype": "float32", "dist": "normal", "seed": 6225, "scale": 0.2 },
"valueT": { "shape": [1, 512, 512], "dtype": "float32", "dist": "normal", "seed": 6226, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 32, 512], "dtype": "float32", "dist": "empty" } },
"bench": {
"primary": true,
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" }
]
}
},
{
"name": "materialized-sgmat-q768-kv772-h4-d80-general-tails",
"preset": "model",
"vars": { "batch": 1, "qSeq": 768, "kvSeq": 772, "heads": 4, "headDim": 80 },
"attrs": { "num_heads": 4, "scale": 0.11180339887498948 },
"inputs": {
"queryT": { "shape": [1, 768, 320], "dtype": "float32", "dist": "normal", "seed": 6331, "scale": 0.2 },
"keyT": { "shape": [1, 772, 320], "dtype": "float32", "dist": "normal", "seed": 6332, "scale": 0.2 },
"valueT": { "shape": [1, 772, 320], "dtype": "float32", "dist": "normal", "seed": 6333, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 768, 320], "dtype": "float32" } },
"bench": {
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" }
]
}
},
{
"name": "materialized-portable-cached-q512-kv512-h1-d128",
"preset": "model",
"vars": { "batch": 1, "qSeq": 512, "kvSeq": 512, "heads": 1, "headDim": 128 },
"attrs": { "num_heads": 1, "scale": 0.08838834764831845 },
"tunableSpace": { "MATERIALIZED_CACHED_SOFTMAX_WORKGROUP_SIZE": [64, 128, 256] },
"inputs": {
"queryT": { "shape": [1, 512, 128], "dtype": "float32", "dist": "normal", "seed": 6341, "scale": 0.2 },
"keyT": { "shape": [1, 512, 128], "dtype": "float32", "dist": "normal", "seed": 6342, "scale": 0.2 },
"valueT": { "shape": [1, 512, 128], "dtype": "float32", "dist": "normal", "seed": 6343, "scale": 0.2 }
},
"outputs": { "outputT": { "shape": [1, 512, 128], "dtype": "float32" } },
"bench": {
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" }
]
}
},
{
"name": "materialized-portable-cached-q512-kv512-h1-d64-bias",
"preset": "model",
"vars": { "batch": 1, "qSeq": 512, "kvSeq": 512, "heads": 1, "headDim": 64 },
"attrs": { "num_heads": 1, "scale": 0.125 },
"tunableSpace": { "MATERIALIZED_CACHED_SOFTMAX_WORKGROUP_SIZE": [64, 128, 256] },
"inputs": {
"queryT": { "shape": [1, 512, 64], "dtype": "float32", "dist": "normal", "seed": 6351, "scale": 0.2 },
"keyT": { "shape": [1, 512, 64], "dtype": "float32", "dist": "normal", "seed": 6352, "scale": 0.2 },
"valueT": { "shape": [1, 512, 64], "dtype": "float32", "dist": "normal", "seed": 6353, "scale": 0.2 },
"biasT": { "shape": [192], "dtype": "float32", "dist": "normal", "seed": 6354, "scale": 0.05 }
},
"outputs": { "outputT": { "shape": [1, 512, 64], "dtype": "float32" } },
"bench": {
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" }
]
}
},
{
"name": "prefill-f32-bias-q128-kv128-h4-d64",
"preset": "model",
"provenance": {
"notes": "Lower f32 fused-bias register-topology boundary comparing the Q32-broadcast and subgroup-cluster routes."
},
"vars": { "batch": 1, "qSeq": 128, "kvSeq": 128, "heads": 4, "headDim": 64 },
"attrs": { "num_heads": 4, "scale": 0.125 },
"inputs": {
"queryT": { "shape": [1, 128, 256], "dtype": "float32", "dist": "normal", "seed": 6355, "scale": 0.2 },
"keyT": { "shape": [1, 128, 256], "dtype": "float32", "dist": "normal", "seed": 6356, "scale": 0.2 },
"valueT": { "shape": [1, 128, 256], "dtype": "float32", "dist": "normal", "seed": 6357, "scale": 0.2 },
"biasT": { "shape": [768], "dtype": "float32", "dist": "normal", "seed": 6358, "scale": 0.05 }
},
"outputs": { "outputT": { "shape": [1, 128, 256], "dtype": "float32" } },
"bench": {
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" }
]
}
},
{
"name": "prefill-f32-bias-q128-kv128-h4-d256",
"preset": "model",
"provenance": {
"notes": "Upper f32 fused-bias register-topology boundary comparing the Q32-broadcast and subgroup-cluster routes."
},
"vars": { "batch": 1, "qSeq": 128, "kvSeq": 128, "heads": 4, "headDim": 256 },
"attrs": { "num_heads": 4, "scale": 0.0625 },
"inputs": {
"queryT": { "shape": [1, 128, 1024], "dtype": "float32", "dist": "normal", "seed": 6361, "scale": 0.2 },
"keyT": { "shape": [1, 128, 1024], "dtype": "float32", "dist": "normal", "seed": 6362, "scale": 0.2 },
"valueT": { "shape": [1, 128, 1024], "dtype": "float32", "dist": "normal", "seed": 6363, "scale": 0.2 },
"biasT": { "shape": [3072], "dtype": "float32", "dist": "normal", "seed": 6364, "scale": 0.05 }
},
"outputs": { "outputT": { "shape": [1, 128, 1024], "dtype": "float32" } },
"bench": {
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" }
]
}
},
{
"name": "mha-f32-q32-kv512-h8-d64-attn-bias-flash",
"preset": "stress",
"vars": { "batch": 1, "qSeq": 32, "kvSeq": 512, "heads": 8, "headDim": 64 },
"attrs": { "num_heads": 8, "scale": 0.125 },
"inputs": {
"queryT": { "shape": [1, 32, 512], "dtype": "float32", "dist": "normal", "seed": 6224, "scale": 0.2 },
"keyT": { "shape": [1, 512, 512], "dtype": "float32", "dist": "normal", "seed": 6225, "scale": 0.2 },
"valueT": { "shape": [1, 512, 512], "dtype": "float32", "dist": "normal", "seed": 6226, "scale": 0.2 },
"attentionBiasT": { "shape": [1, 1, 32, 512], "dtype": "float32", "dist": "normal", "seed": 471, "scale": 0.3 }
},
"outputs": { "outputT": { "shape": [1, 32, 512], "dtype": "float32", "dist": "empty" } },
"bench": {
"primary": true,
"metrics": [
{ "type": "gflops", "value": "4 * args.batch * args.qSeq * args.kvSeq * args.heads * args.headDim" }
]
}
}
]
}
|