diff --git "a/build/webgpu/test.json" "b/build/webgpu/test.json" new file mode 100644--- /dev/null +++ "b/build/webgpu/test.json" @@ -0,0 +1,4660 @@ +{ + "op": "com.microsoft.GroupQueryAttention", + "fixtureArrays": { + "ort_past_key_t_pattern": [0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8], + "ort_past_value_t_pattern": [0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3], + "ort_query_t_pattern": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2], + "ort_large_past_key_t_pattern": [0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2], + "ort_large_past_value_t_pattern": [0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3], + "ort_rotary_cos_cache_t": [1, 1, 1, 1, 1, 1, 1, 1, 0.5403023, 0.9504153, 0.9950042, 0.9995, 0.99995, 0.999995, 0.9999995, 1, -0.4161468, 0.8065784, 0.9800666, 0.9980007, 0.9998, 0.99998, 0.999998, 0.9999998, -0.9899925, 0.5827536, 0.9553365, 0.9955034, 0.99955, 0.999955, 0.9999955, 0.9999996, -0.6536436, 0.3011375, 0.921061, 0.9920107, 0.9992001, 0.99992, 0.999992, 0.9999992, 0.2836622, -0.0103423, 0.8775826, 0.987526, 0.9987503, 0.999875, 0.9999875, 0.9999988, 0.9601703, -0.3207965, 0.8253356, 0.9820539, 0.9982005, 0.99982, 0.999982, 0.9999982, 0.7539023, -0.5994374, 0.7648422, 0.9755999, 0.997551, 0.999755, 0.9999755, 0.9999976, -0.1455, -0.8186325, 0.6967067, 0.9681703, 0.9968017, 0.99968, 0.999968, 0.9999968, -0.9111303, -0.9566442, 0.62161, 0.9597726, 0.9959527, 0.999595, 0.9999595, 0.999996, -0.8390715, -0.9997861, 0.5403023, 0.9504153, 0.9950042, 0.9995, 0.99995, 0.999995, 0.0044257, -0.9437797, 0.4535961, 0.9401076, 0.9939561, 0.9993951, 0.9999395, 0.999994, 0.843854, -0.7941793, 0.3623578, 0.9288599, 0.9928086, 0.9992801, 0.999928, 0.9999928, 0.9074468, -0.5658205, 0.2674988, 0.9166834, 0.9915619, 0.9991551, 0.9999155, 0.9999916, 0.1367372, -0.2813496, 0.1699671, 0.9035902, 0.990216, 0.9990202, 0.999902, 0.9999902, -0.7596879, 0.0310225, 0.0707372, 0.8895936, 0.9887711, 0.9988752, 0.9998875, 0.9999888, -0.9576595, 0.3403182, -0.0291995, 0.8747075, 0.9872273, 0.9987203, 0.999872, 0.9999872, -0.2751633, 0.6158647, -0.1288445, 0.8589467, 0.9855848, 0.9985553, 0.9998555, 0.9999856, 0.6603167, 0.8303362, -0.2272021, 0.842327, 0.9838437, 0.9983804, 0.999838, 0.9999838, 0.9887046, 0.9624638, -0.3232896, 0.8248651, 0.9820042, 0.9981955, 0.9998195, 0.999982, 0.4080821, 0.9991444, -0.4161468, 0.8065784, 0.9800666, 0.9980007, 0.9998, 0.99998, -0.5477293, 0.9367404, -0.5048461, 0.7874852, 0.9780309, 0.9977958, 0.9997795, 0.999978, -0.9999608, 0.7814403, -0.5885011, 0.7676045, 0.9758974, 0.997581, 0.999758, 0.9999758, -0.532833, 0.5486453, -0.666276, 0.7469564, 0.9736664, 0.9973562, 0.9997355, 0.9999736], + "ort_rotary_sin_cache_t": [0, 0, 0, 0, 0, 0, 0, 0, 0.841471, 0.3109836, 0.0998334, 0.0316175, 0.0099998, 0.0031623, 0.001, 0.0003162, 0.9092974, 0.5911271, 0.1986693, 0.0632034, 0.0199987, 0.0063245, 0.002, 0.0006325, 0.14112, 0.8126489, 0.2955202, 0.0947261, 0.0299955, 0.0094867, 0.003, 0.0009487, -0.7568025, 0.9535807, 0.3894183, 0.1261541, 0.0399893, 0.0126488, 0.004, 0.0012649, -0.9589243, 0.9999465, 0.4794255, 0.1574559, 0.0499792, 0.0158107, 0.005, 0.0015811, -0.2794155, 0.9471482, 0.5646425, 0.1886003, 0.059964, 0.0189725, 0.006, 0.0018974, 0.6569866, 0.8004216, 0.6442177, 0.2195561, 0.0699428, 0.0221341, 0.0069999, 0.0022136, 0.9893582, 0.5743178, 0.7173561, 0.2502924, 0.0799147, 0.0252955, 0.0079999, 0.0025298, 0.4121185, 0.2912591, 0.7833269, 0.2807784, 0.0898785, 0.0284567, 0.0089999, 0.002846, -0.5440211, -0.0206835, 0.841471, 0.3109836, 0.0998334, 0.0316175, 0.0099998, 0.0031623, -0.9999902, -0.330575, 0.8912074, 0.3408779, 0.1097783, 0.034778, 0.0109998, 0.0034785, -0.5365729, -0.6076835, 0.9320391, 0.3704313, 0.1197122, 0.0379382, 0.0119997, 0.0037947, 0.420167, -0.8245285, 0.9635582, 0.3996143, 0.1296341, 0.041098, 0.0129996, 0.0041109, 0.9906074, -0.9596053, 0.9854497, 0.4283978, 0.1395431, 0.0442574, 0.0139995, 0.0044272, 0.6502878, -0.9995187, 0.997495, 0.4567529, 0.1494381, 0.0474164, 0.0149994, 0.0047434, -0.2879033, -0.9403103, 0.9995736, 0.4846513, 0.1593182, 0.0505749, 0.0159993, 0.0050596, -0.9613975, -0.7878519, 0.9916648, 0.512065, 0.1691823, 0.0537328, 0.0169992, 0.0053758, -0.7509872, -0.5572627, 0.9738476, 0.5389668, 0.1790296, 0.0568903, 0.017999, 0.0056921, 0.1498772, -0.27141, 0.9463001, 0.5653296, 0.1888589, 0.0600471, 0.0189989, 0.0060083, 0.9129453, 0.0413582, 0.9092974, 0.5911271, 0.1986693, 0.0632034, 0.0199987, 0.0063245, 0.8366556, 0.350025, 0.8632094, 0.6163336, 0.2084599, 0.066359, 0.0209985, 0.0066407, -0.0088513, 0.62398, 0.8084964, 0.6409238, 0.2182296, 0.069514, 0.0219982, 0.006957, -0.8462204, 0.8360552, 0.7457052, 0.6648731, 0.2279775, 0.0726683, 0.022998, 0.0072732], + "ort_quant_query_t": [0.082308, 0.019819, -0.034024, 0.277037, -0.077135, -0.466628, -0.082611, 0.308729, 0.11234, 0.214904, -0.317721, 0.016195, 0.061967, -0.251316, 0.419605, -0.389136, 0.158377, -0.171533, -0.314711, 0.171006, 0.418748, -0.368512, -0.089387, 0.349135, -0.090375, -0.467301, 0.066158, 0.304013, -0.351828, 0.402985, -0.01996, -0.048985, 0.204581, 0.335118, 0.279197, 0.124608, -0.275582, 0.141191, 0.422468, -0.253317, -0.410373, 0.46529, -0.205779, -0.405497, 0.436971, -0.14485, 0.327278, -0.238, -0.247373, -0.363456, -0.37611, 0.408198, -0.238281, -0.065844, 0.043498, -0.113526, -0.168485, -0.317308, -0.273944, -0.151551, 0.107045, -0.028029, 0.01964, 0.416226], + "ort_quant_key_t": [0.372574, -0.306708, 0.184696, 0.272713, -0.148147, -0.122402, 0.033584, -0.125638, 0.227005, -0.30606, 0.136125, 0.167737, 0.018802, -0.24656, -0.205723, 0.118218, 0.497002, 0.044897, -0.050453, -0.078412, -0.326121, 0.042089, 0.441168, 0.458006, -0.149514, 0.059117, 0.301394, -0.209086, 0.059405, 0.322551, 0.181637, -0.070391], + "ort_quant_value_t": [0.097056, -0.200735, -0.168325, 0.191973, 0.492785, -0.397293, 0.47762, 0.470182, -0.256314, -0.16696, -0.157802, 0.27402, 0.404261, 0.289624, -0.477124, -0.490144, -0.067094, 0.221887, 0.084429, -0.272333, -0.492298, 0.063575, -0.344991, 0.405092, 0.051202, -0.053699, 0.465574, 0.05767, 0.307941, -0.038854, -0.073572, -0.106016], + "ort_quant_int8_large_head_zero_cache": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + "ort_sharedkv_decode_b1q1p8_h2kv1d8_input_queryT": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2], + "ort_sharedkv_batched_b2q1p4_h2kv1d8_input_queryT": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4], + "ort_sharedkv_rotary_decode_b1q1p8_h2kv1d16_input_pastKeyT": [0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6, 0.8, 1, 0.2, 0.4, 0.6], + "ort_sharedkv_rotary_decode_b1q1p8_h2kv1d16_input_pastValueT": [0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6], + "ort_sharedkv_rotary_multibatch_b2q1p8_h2kv1d16_input_queryT": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1], + "ort_quant_int8_pertensor_b1q4_h2kv1d8_input_pastKeyT": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + "ort_quant_int4_pertensor_b1q4_h2kv1d8_input_pastKeyT": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + "ort_quant_int8_multibatch_b2q4_h4kv2d16_input_pastKeyT": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + "cases": [ + { + "name": "present_outputs", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "scale": 0.5, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 3, 12], + "data": { "kind": "fillFloat32", "sinStep": 0.17, "cosStep": 0.31 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 4, 6], + "data": { "kind": "fillFloat32", "sinStep": 0.11, "cosStep": 0.23 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 4, 6], + "data": { "kind": "fillFloat32", "sinStep": 0.07, "cosStep": 0.41 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [3] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 3, 12], "tolerance": 0.00002 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 4, 3], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 4, 3], "tolerance": 0.000001 } + } + }, + { + "name": "direct_f32_with_f16_present_cache", + "provenance": { + "notes": "Connects the independent T and T_CACHE schema variables by narrowing a direct float32 K/V projection into float16 present-cache storage." + }, + "attrs": { "num_heads": 2, "kv_num_heads": 1, "scale": 0.5, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 2, 6], + "data": { "kind": "fillFloat32", "scale": 0.4, "sinStep": 0.17, "cosStep": 0.31 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 3, 3], + "data": { "kind": "fillFloat32", "scale": 0.4, "sinStep": 0.11, "cosStep": 0.23 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 3, 3], + "data": { "kind": "fillFloat32", "scale": 0.4, "sinStep": 0.07, "cosStep": 0.41 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [2] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [3] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 2, 6], "tolerance": 0.00002 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 1, 3, 3], "tolerance": 0.001 }, + "presentValueT": { "dtype": "float16", "shape": [1, 1, 3, 3], "tolerance": 0.001 } + } + }, + { + "name": "direct_f16_with_f32_present_cache", + "provenance": { + "notes": "Exercises the opposite independent T/T_CACHE conversion by widening a direct float16 K/V projection into float32 present-cache storage." + }, + "attrs": { "num_heads": 2, "kv_num_heads": 1, "scale": 0.5, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 2, 6], + "data": { "kind": "fillFloat32", "scale": 0.4, "sinStep": 0.17, "cosStep": 0.31 } + }, + "keyT": { + "dtype": "float16", + "shape": [1, 3, 3], + "data": { "kind": "fillFloat32", "scale": 0.4, "sinStep": 0.11, "cosStep": 0.23 } + }, + "valueT": { + "dtype": "float16", + "shape": [1, 3, 3], + "data": { "kind": "fillFloat32", "scale": 0.4, "sinStep": 0.07, "cosStep": 0.41 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [2] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [3] } } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 2, 6], "tolerance": 0.0005, "relTolerance": 0.002 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 3, 3], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 3, 3], "tolerance": 0.000001 } + } + }, + { + "name": "mqa_four_query_heads_one_kv_head", + "attrs": { "num_heads": 4, "kv_num_heads": 1, "scale": 0.7071067811865476, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 2, 8], + "data": { "kind": "fillFloat32", "scale": 0.5, "sinStep": 0.17, "cosStep": 0.31 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 3, 2], + "data": { "kind": "fillFloat32", "scale": 0.5, "sinStep": 0.11, "cosStep": 0.23 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 3, 2], + "data": { "kind": "fillFloat32", "scale": 0.5, "sinStep": 0.07, "cosStep": 0.41 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [2] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [3] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 2, 8], "tolerance": 0.00002 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 3, 2], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 3, 2], "tolerance": 0.000001 } + } + }, + { + "name": "flash_prefill_h2kv1_d64_s128", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "scale": 0.125, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 128, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.13, "cosStep": 0.29 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 128, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.19, "cosStep": 0.07 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 128, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.23, "cosStep": 0.17 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [127] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [128] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 128, 128], "tolerance": 0.00008 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 128, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 128, 64], "tolerance": 0.000001 } + } + }, + { + "name": "flash_prefill_h4kv1_d256_s128", + "attrs": { "num_heads": 4, "kv_num_heads": 1, "scale": 0.0625, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 128, 1024], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.13, "cosStep": 0.29 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 128, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.19, "cosStep": 0.07 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 128, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.23, "cosStep": 0.17 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [127] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [128] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 128, 1024], "tolerance": 0.00008 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 128, 256], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 128, 256], "tolerance": 0.000001 } + }, + "provenance": { + "notes": "At headDim 256, a four-lane-per-query layout would require 16 vec4 values per lane, so the slice cap raises lanes per query to eight. Four query heads across 128 tokens keep the wide clustered geometry occupied while checking the register-slice bound." + } + }, + { + "name": "flash_q32_broadcast_f32_h8kv2_d128_s32", + "provenance": { + "notes": "Compact f32 Q32 broadcast coverage for the low-storage subgroup path: 32 vec4 head segments equal the fixed 32-lane subgroup." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.08838834764831845, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 32, 1024], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 32, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 32, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 32, 1024], "tolerance": 0.0001, "relTolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 32, 128], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 32, 128], "tolerance": 0.000001 } + } + }, + { + "name": "flash_decode_h4kv1_d64_kv512", + "attrs": { "num_heads": 4, "kv_num_heads": 1, "scale": 0.125, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 256], + "data": { "kind": "fillFloat32", "scale": 0.18, "sinStep": 0.13, "cosStep": 0.29 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 512, 64], + "data": { "kind": "fillFloat32", "scale": 0.18, "sinStep": 0.19, "cosStep": 0.07 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 512, 64], + "data": { "kind": "fillFloat32", "scale": 0.18, "sinStep": 0.23, "cosStep": 0.17 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [511] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [512] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 256], "tolerance": 0.00008 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 512, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 512, 64], "tolerance": 0.000001 } + } + }, + { + "name": "multi_batch_gqa_two_kv_heads_head4", + "provenance": { + "source": "onnxruntime/test/contrib_ops/group_query_attention_op_test.cc", + "notes": "Exercises multi-batch GQA with q_heads/kv_heads=2 inside the simplified no-cache fixture form." + }, + "attrs": { "num_heads": 4, "kv_num_heads": 2, "scale": 0.5, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [2, 2, 16], + "data": { "kind": "fillFloat32", "scale": 0.35, "sinStep": 0.13, "cosStep": 0.29 } + }, + "keyT": { + "dtype": "float32", + "shape": [2, 3, 8], + "data": { "kind": "fillFloat32", "scale": 0.4, "sinStep": 0.19, "cosStep": 0.07 } + }, + "valueT": { + "dtype": "float32", + "shape": [2, 3, 8], + "data": { "kind": "fillFloat32", "scale": 0.45, "sinStep": 0.23, "cosStep": 0.17 } + }, + "seqlensKT": { "dtype": "int32", "shape": [2], "data": { "kind": "values", "values": [2, 2] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [3] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [2, 2, 16], "tolerance": 0.00003 }, + "presentKeyT": { "dtype": "float32", "shape": [2, 2, 3, 4], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [2, 2, 3, 4], "tolerance": 0.000001 } + } + }, + { + "name": "mha_equal_query_and_kv_heads", + "provenance": { + "source": "onnxruntime/test/contrib_ops/group_query_attention_op_test.cc", + "notes": "Covers the q_heads == kv_heads boundary where GroupQueryAttention degenerates to ordinary multi-head attention." + }, + "attrs": { "num_heads": 2, "kv_num_heads": 2, "scale": 0.5, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 3, 8], + "data": { + "kind": "values", + "values": [0.2, -0.1, 0.5, 0.7, -0.3, 0.4, -0.6, 0.1, 0.9, -0.8, 0.2, -0.4, 0.3, 0.6, -0.7, 0.5, -0.5, 0.1, 0.8, -0.2, 0.7, -0.9, 0.4, 0.3] + } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 2, 8], + "data": { + "kind": "values", + "values": [0.1, 0.3, -0.5, 0.2, 0.4, -0.6, 0.8, -0.1, -0.7, 0.9, 0.2, 0.5, -0.3, 0.1, -0.4, 0.6] + } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 2, 8], + "data": { + "kind": "values", + "values": [1.0, -1.0, 2.0, -2.0, 0.5, 0.25, -0.5, -0.25, -1.5, 1.5, 0.75, -0.75, 2.5, -2.5, 0.125, -0.125] + } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [1] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [2] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 3, 8], "tolerance": 0.00002 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 2, 4], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 2, 4], "tolerance": 0.000001 } + } + }, + { + "name": "single_query_longer_kv_mqa_head4", + "provenance": { + "source": "onnxruntime/test/contrib_ops/group_query_attention_op_test.cc", + "notes": "MQA path with one KV head, q_seq=1, and a longer KV sequence." + }, + "attrs": { "num_heads": 4, "kv_num_heads": 1, "scale": 0.5, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 16], + "data": { "kind": "fillFloat32", "scale": 0.25, "sinStep": 0.41, "cosStep": 0.09 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 5, 4], + "data": { "kind": "fillFloat32", "scale": 0.3, "sinStep": 0.17, "cosStep": 0.37 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 5, 4], + "data": { "kind": "fillFloat32", "scale": 0.5, "sinStep": 0.31, "cosStep": 0.11 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [5] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 16], "tolerance": 0.00002 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 5, 4], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 5, 4], "tolerance": 0.000001 } + } + }, + { + "name": "ort_webgpu_shared_kv_decode_projection", + "provenance": { + "source": "onnxruntime/test/contrib_ops/group_query_attention_op_test.cc", + "test": "GroupQueryAttentionTest.WebGPU_SharedKV_Decode", + "notes": "Projects ORT's shared-KV decode data pattern into the simplified no-past fixture form with q_seq=1, kv_seq=8, q_heads=2, kv_heads=1, head_size=8." + }, + "attrs": { "num_heads": 2, "kv_num_heads": 1, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 16], + "data": { "kind": "cycle", "values": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7] } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 8, 8], + "data": { "kind": "cycle", "values": [0.2, 0.4, 0.6, 0.8, 1.0] } + }, + "valueT": { "dtype": "float32", "shape": [1, 8, 8], "data": { "kind": "cycle", "values": [0.3, 0.6, 0.9] } }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [7] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [8] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 16], "tolerance": 0.00003 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.000001 } + } + }, + { + "name": "ort_webgpu_shared_kv_prefill_projection", + "provenance": { + "source": "onnxruntime/test/contrib_ops/group_query_attention_op_test.cc", + "test": "GroupQueryAttentionTest.WebGPU_SharedKV_Prefill", + "notes": "Projects ORT's shared-KV prefill pattern into the simplified no-past fixture form with q_seq=8 and the same MQA head layout." + }, + "attrs": { "num_heads": 2, "kv_num_heads": 1, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 8, 16], + "data": { "kind": "cycle", "values": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7] } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 8, 8], + "data": { "kind": "cycle", "values": [0.2, 0.4, 0.6, 0.8, 1.0] } + }, + "valueT": { "dtype": "float32", "shape": [1, 8, 8], "data": { "kind": "cycle", "values": [0.3, 0.6, 0.9] } }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [7] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [8] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 8, 16], "tolerance": 0.00003 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.000001 } + } + }, + { + "name": "ort_webgpu_shared_kv_sliding_window_projection", + "provenance": { + "source": "onnxruntime/test/contrib_ops/group_query_attention_op_test.cc", + "test": "GroupQueryAttentionTest.WebGPU_SharedKV_SlidingWindow", + "notes": "Projects ORT's long shared-KV WebGPU sliding-window stress shape into the simplified no-past fixture form." + }, + "attrs": { "num_heads": 2, "kv_num_heads": 1, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 4, 16], + "data": { "kind": "cycle", "values": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7] } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 32, 8], + "data": { "kind": "cycle", "values": [0.2, 0.4, 0.6, 0.8, 1.0] } + }, + "valueT": { "dtype": "float32", "shape": [1, 32, 8], "data": { "kind": "cycle", "values": [0.3, 0.6, 0.9] } }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 4, 16], "tolerance": 0.00003 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 32, 8], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 32, 8], "tolerance": 0.000001 } + } + }, + { + "name": "splitk_decode_h8kv2_d64_kv2048", + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 512], + "data": { "kind": "fillFloat32", "scale": 0.18, "sinStep": 0.013, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 2048, 128], + "data": { "kind": "fillFloat32", "scale": 0.18, "sinStep": 0.019, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 2048, 128], + "data": { "kind": "fillFloat32", "scale": 0.18, "sinStep": 0.023, "cosStep": 0.017 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [2047] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [2048] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 512], "tolerance": 0.00008 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 2048, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 2048, 64], "tolerance": 0.000001 } + } + }, + { + "name": "splitk_decode_h4kv1_d64_kv1024", + "attrs": { "num_heads": 4, "kv_num_heads": 1, "scale": 0.125, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 256], + "data": { "kind": "fillFloat32", "scale": 0.18, "sinStep": 0.013, "cosStep": 0.031 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1024, 64], + "data": { "kind": "fillFloat32", "scale": 0.18, "sinStep": 0.019, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1024, 64], + "data": { "kind": "fillFloat32", "scale": 0.18, "sinStep": 0.023, "cosStep": 0.017 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [1023] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [1024] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 256], "tolerance": 0.00008 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 1024, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 1024, 64], "tolerance": 0.000001 } + } + }, + { + "name": "flash_prefill_h2kv1_d64_s128_f16", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "scale": 0.125, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 128, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.13, "cosStep": 0.29 } + }, + "keyT": { + "dtype": "float16", + "shape": [1, 128, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.19, "cosStep": 0.07 } + }, + "valueT": { + "dtype": "float16", + "shape": [1, 128, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.23, "cosStep": 0.17 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [127] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [128] } } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 128, 128], "tolerance": 0.03 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 1, 128, 64], "tolerance": 0.03 }, + "presentValueT": { "dtype": "float16", "shape": [1, 1, 128, 64], "tolerance": 0.03 } + } + }, + { + "name": "flash_decode_h4kv1_d64_kv512_f16", + "attrs": { "num_heads": 4, "kv_num_heads": 1, "scale": 0.125, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 1, 256], + "data": { "kind": "fillFloat32", "scale": 0.18, "sinStep": 0.13, "cosStep": 0.29 } + }, + "keyT": { + "dtype": "float16", + "shape": [1, 512, 64], + "data": { "kind": "fillFloat32", "scale": 0.18, "sinStep": 0.19, "cosStep": 0.07 } + }, + "valueT": { + "dtype": "float16", + "shape": [1, 512, 64], + "data": { "kind": "fillFloat32", "scale": 0.18, "sinStep": 0.23, "cosStep": 0.17 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [511] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [512] } } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 1, 256], "tolerance": 0.03 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 1, 512, 64], "tolerance": 0.03 }, + "presentValueT": { "dtype": "float16", "shape": [1, 1, 512, 64], "tolerance": 0.03 } + } + }, + { + "name": "splitk_decode_h8kv2_d64_kv2048_f16", + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 1, 512], + "data": { "kind": "fillFloat32", "scale": 0.18, "sinStep": 0.013, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float16", + "shape": [1, 2048, 128], + "data": { "kind": "fillFloat32", "scale": 0.18, "sinStep": 0.019, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float16", + "shape": [1, 2048, 128], + "data": { "kind": "fillFloat32", "scale": 0.18, "sinStep": 0.023, "cosStep": 0.017 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [2047] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [2048] } } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 1, 512], "tolerance": 0.03 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 2, 2048, 64], "tolerance": 0.03 }, + "presentValueT": { "dtype": "float16", "shape": [1, 2, 2048, 64], "tolerance": 0.03 } + } + }, + { + "name": "sliding_window_w3_scalar_h2kv1_d8_s6", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "local_window_size": 3 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 6, 16], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.11, "cosStep": 0.27 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 6, 8], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.17, "cosStep": 0.19 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 6, 8], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.23, "cosStep": 0.13 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [5] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [6] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 6, 16], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 6, 8], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 6, 8], "tolerance": 0.000001 } + } + }, + { + "name": "sliding_window_w16_cluster_h8kv2_d64_s64", + "attrs": { "num_heads": 8, "kv_num_heads": 2, "local_window_size": 16 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 64, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 64, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 64, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [63] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [64] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 64, 512], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 64, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 64, 64], "tolerance": 0.000001 } + } + }, + { + "name": "sliding_window_w8_online_h16kv4_d64_s16", + "attrs": { "num_heads": 16, "kv_num_heads": 4, "local_window_size": 8 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 16, 1024], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.013, "cosStep": 0.031 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 16, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.019, "cosStep": 0.017 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 16, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.029, "cosStep": 0.011 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [15] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [16] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 16, 1024], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 4, 16, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 4, 16, 64], "tolerance": 0.000001 } + } + }, + { + "name": "sliding_window_w256_decode_h8kv2_d64_kv1024", + "attrs": { "num_heads": 8, "kv_num_heads": 2, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1024, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.0017, "cosStep": 0.0019 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1024, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.0023, "cosStep": 0.0013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [1023] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [1024] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 512], "tolerance": 0.0002 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 1024, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 1024, 64], "tolerance": 0.000001 } + } + }, + { + "name": "ort_sharedkv_decode_b1q1p8_h2kv1d8", + "attrs": { "num_heads": 2, "kv_num_heads": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 16], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_sharedkv_decode_b1q1p8_h2kv1d8_input_queryT" } + } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 8], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 8], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_key_t_pattern" } } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_value_t_pattern" } } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [7] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [8] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 16], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 } + } + }, + { + "name": "ort_sharedkv_prompt_b1q8p8_h2kv1d8", + "attrs": { "num_heads": 2, "kv_num_heads": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 8, 16], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_query_t_pattern" } } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 8], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 8], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_key_t_pattern" } } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_value_t_pattern" } } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [7] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [8] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 8, 16], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 } + } + }, + { + "name": "ort_sharedkv_largehead_b1q1p4_h2kv1d64", + "attrs": { "num_heads": 2, "kv_num_heads": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_query_t_pattern" } } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 64], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 64], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 4, 64], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_large_past_key_t_pattern" } } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 4, 64], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_large_past_value_t_pattern" } } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [3] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 128], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 4, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 4, 64], "tolerance": 0.0001 } + } + }, + { + "name": "ort_sharedkv_gqaratio8_b1q1p4_h8kv1d16", + "attrs": { "num_heads": 8, "kv_num_heads": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_query_t_pattern" } } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 16], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 16], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 4, 16], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_key_t_pattern" } } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 4, 16], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_value_t_pattern" } } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [3] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 128], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 4, 16], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 4, 16], "tolerance": 0.0001 } + } + }, + { + "name": "ort_sharedkv_batched_b2q1p4_h2kv1d8", + "attrs": { "num_heads": 2, "kv_num_heads": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [2, 1, 16], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_sharedkv_batched_b2q1p4_h2kv1d8_input_queryT" } + } + }, + "keyT": { "dtype": "float32", "shape": [2, 0, 8], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [2, 0, 8], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [2, 1, 4, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_key_t_pattern" } } + }, + "pastValueT": { + "dtype": "float32", + "shape": [2, 1, 4, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_value_t_pattern" } } + }, + "seqlensKT": { "dtype": "int32", "shape": [2], "data": { "kind": "values", "values": [3, 3] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [2, 1, 16], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [2, 1, 4, 8], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [2, 1, 4, 8], "tolerance": 0.0001 } + } + }, + { + "name": "ort_sharedkv_rotary_decode_b1q1p8_h2kv1d16", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "do_rotary": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 32], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_sharedkv_batched_b2q1p4_h2kv1d8_input_queryT" } + } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 16], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 16], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 8, 16], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_sharedkv_rotary_decode_b1q1p8_h2kv1d16_input_pastKeyT" } + } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 8, 16], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_sharedkv_rotary_decode_b1q1p8_h2kv1d16_input_pastValueT" } + } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [7] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [8] } }, + "cosCacheT": { + "dtype": "float32", + "shape": [24, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_rotary_cos_cache_t" } } + }, + "sinCacheT": { + "dtype": "float32", + "shape": [24, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_rotary_sin_cache_t" } } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 32], "tolerance": 0.001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 8, 16], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 8, 16], "tolerance": 0.0001 } + } + }, + { + "name": "ort_sharedkv_rotary_prefill_b1q4p4_h2kv1d16", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "do_rotary": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 4, 32], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_query_t_pattern" } } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 16], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 16], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 4, 16], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_key_t_pattern" } } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 4, 16], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_value_t_pattern" } } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [3] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } }, + "cosCacheT": { + "dtype": "float32", + "shape": [20, 8], + "data": { + "kind": "values", + "values": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.5403023, 0.9504153, 0.9950042, 0.9995, 0.99995, 0.999995, 0.9999995, 1.0, -0.4161468, 0.8065784, 0.9800666, 0.9980007, 0.9998, 0.99998, 0.999998, 0.9999998, -0.9899925, 0.5827536, 0.9553365, 0.9955034, 0.99955, 0.999955, 0.9999955, 0.9999996, -0.6536436, 0.3011375, 0.921061, 0.9920107, 0.9992001, 0.99992, 0.999992, 0.9999992, 0.2836622, -0.0103423, 0.8775826, 0.987526, 0.9987503, 0.999875, 0.9999875, 0.9999988, 0.9601703, -0.3207965, 0.8253356, 0.9820539, 0.9982005, 0.99982, 0.999982, 0.9999982, 0.7539023, -0.5994374, 0.7648422, 0.9755999, 0.997551, 0.999755, 0.9999755, 0.9999976, -0.1455, -0.8186325, 0.6967067, 0.9681703, 0.9968017, 0.99968, 0.999968, 0.9999968, -0.9111303, -0.9566442, 0.62161, 0.9597726, 0.9959527, 0.999595, 0.9999595, 0.999996, -0.8390715, -0.9997861, 0.5403023, 0.9504153, 0.9950042, 0.9995, 0.99995, 0.999995, 0.0044257, -0.9437797, 0.4535961, 0.9401076, 0.9939561, 0.9993951, 0.9999395, 0.999994, 0.843854, -0.7941793, 0.3623578, 0.9288599, 0.9928086, 0.9992801, 0.999928, 0.9999928, 0.9074468, -0.5658205, 0.2674988, 0.9166834, 0.9915619, 0.9991551, 0.9999155, 0.9999916, 0.1367372, -0.2813496, 0.1699671, 0.9035902, 0.990216, 0.9990202, 0.999902, 0.9999902, -0.7596879, 0.0310225, 0.0707372, 0.8895936, 0.9887711, 0.9988752, 0.9998875, 0.9999888, -0.9576595, 0.3403182, -0.0291995, 0.8747075, 0.9872273, 0.9987203, 0.999872, 0.9999872, -0.2751633, 0.6158647, -0.1288445, 0.8589467, 0.9855848, 0.9985553, 0.9998555, 0.9999856, 0.6603167, 0.8303362, -0.2272021, 0.842327, 0.9838437, 0.9983804, 0.999838, 0.9999838, 0.9887046, 0.9624638, -0.3232896, 0.8248651, 0.9820042, 0.9981955, 0.9998195, 0.999982] + } + }, + "sinCacheT": { + "dtype": "float32", + "shape": [20, 8], + "data": { + "kind": "values", + "values": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.841471, 0.3109836, 0.0998334, 0.0316175, 0.0099998, 0.0031623, 0.001, 0.0003162, 0.9092974, 0.5911271, 0.1986693, 0.0632034, 0.0199987, 0.0063245, 0.002, 0.0006325, 0.14112, 0.8126489, 0.2955202, 0.0947261, 0.0299955, 0.0094867, 0.003, 0.0009487, -0.7568025, 0.9535807, 0.3894183, 0.1261541, 0.0399893, 0.0126488, 0.004, 0.0012649, -0.9589243, 0.9999465, 0.4794255, 0.1574559, 0.0499792, 0.0158107, 0.005, 0.0015811, -0.2794155, 0.9471482, 0.5646425, 0.1886003, 0.059964, 0.0189725, 0.006, 0.0018974, 0.6569866, 0.8004216, 0.6442177, 0.2195561, 0.0699428, 0.0221341, 0.0069999, 0.0022136, 0.9893582, 0.5743178, 0.7173561, 0.2502924, 0.0799147, 0.0252955, 0.0079999, 0.0025298, 0.4121185, 0.2912591, 0.7833269, 0.2807784, 0.0898785, 0.0284567, 0.0089999, 0.002846, -0.5440211, -0.0206835, 0.841471, 0.3109836, 0.0998334, 0.0316175, 0.0099998, 0.0031623, -0.9999902, -0.330575, 0.8912074, 0.3408779, 0.1097783, 0.034778, 0.0109998, 0.0034785, -0.5365729, -0.6076835, 0.9320391, 0.3704313, 0.1197122, 0.0379382, 0.0119997, 0.0037947, 0.420167, -0.8245285, 0.9635582, 0.3996143, 0.1296341, 0.041098, 0.0129996, 0.0041109, 0.9906074, -0.9596053, 0.9854497, 0.4283978, 0.1395431, 0.0442574, 0.0139995, 0.0044272, 0.6502878, -0.9995187, 0.997495, 0.4567529, 0.1494381, 0.0474164, 0.0149994, 0.0047434, -0.2879033, -0.9403103, 0.9995736, 0.4846513, 0.1593182, 0.0505749, 0.0159993, 0.0050596, -0.9613975, -0.7878519, 0.9916648, 0.512065, 0.1691823, 0.0537328, 0.0169992, 0.0053758, -0.7509872, -0.5572627, 0.9738476, 0.5389668, 0.1790296, 0.0568903, 0.017999, 0.0056921, 0.1498772, -0.27141, 0.9463001, 0.5653296, 0.1888589, 0.0600471, 0.0189989, 0.0060083] + } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 4, 32], "tolerance": 0.001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 4, 16], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 4, 16], "tolerance": 0.0001 } + } + }, + { + "name": "ort_sharedkv_rotary_multibatch_b2q1p8_h2kv1d16", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "do_rotary": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [2, 1, 32], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_sharedkv_rotary_multibatch_b2q1p8_h2kv1d16_input_queryT" } + } + }, + "keyT": { "dtype": "float32", "shape": [2, 0, 16], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [2, 0, 16], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [2, 1, 8, 16], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_large_past_key_t_pattern" } } + }, + "pastValueT": { + "dtype": "float32", + "shape": [2, 1, 8, 16], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_large_past_value_t_pattern" } } + }, + "seqlensKT": { "dtype": "int32", "shape": [2], "data": { "kind": "values", "values": [7, 7] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [8] } }, + "cosCacheT": { + "dtype": "float32", + "shape": [24, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_rotary_cos_cache_t" } } + }, + "sinCacheT": { + "dtype": "float32", + "shape": [24, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_rotary_sin_cache_t" } } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [2, 1, 32], "tolerance": 0.001 }, + "presentKeyT": { "dtype": "float32", "shape": [2, 1, 8, 16], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [2, 1, 8, 16], "tolerance": 0.0001 } + } + }, + { + "name": "ort_qknorm_rotary_decode_b1q1p4_h2kv1d16", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "do_rotary": 1, "qk_norm_epsilon": 0.00001 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 32], + "data": { + "kind": "values", + "values": [0.03, 0.06, 0.09, 0.12, 0.15, 0.18, 0.21, 0.24, 0.27, 0.3, 0.33, 0.36, 0.39, 0.42, 0.45, 0.48, 0.51, 0.54, 0.57, 0.6, 0.63, 0.66, 0.69, 0.72, 0.75, 0.78, 0.81, 0.84, 0.87, 0.9, 0.93, 0.96] + } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 16], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 16], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 4, 16], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_key_t_pattern" } } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 4, 16], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_value_t_pattern" } } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [3] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } }, + "cosCacheT": { + "dtype": "float32", + "shape": [12, 8], + "data": { + "kind": "values", + "values": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.5403023, 0.9504153, 0.9950042, 0.9995, 0.99995, 0.999995, 0.9999995, 1.0, -0.4161468, 0.8065784, 0.9800666, 0.9980007, 0.9998, 0.99998, 0.999998, 0.9999998, -0.9899925, 0.5827536, 0.9553365, 0.9955034, 0.99955, 0.999955, 0.9999955, 0.9999996, -0.6536436, 0.3011375, 0.921061, 0.9920107, 0.9992001, 0.99992, 0.999992, 0.9999992, 0.2836622, -0.0103423, 0.8775826, 0.987526, 0.9987503, 0.999875, 0.9999875, 0.9999988, 0.9601703, -0.3207965, 0.8253356, 0.9820539, 0.9982005, 0.99982, 0.999982, 0.9999982, 0.7539023, -0.5994374, 0.7648422, 0.9755999, 0.997551, 0.999755, 0.9999755, 0.9999976, -0.1455, -0.8186325, 0.6967067, 0.9681703, 0.9968017, 0.99968, 0.999968, 0.9999968, -0.9111303, -0.9566442, 0.62161, 0.9597726, 0.9959527, 0.999595, 0.9999595, 0.999996, -0.8390715, -0.9997861, 0.5403023, 0.9504153, 0.9950042, 0.9995, 0.99995, 0.999995, 0.0044257, -0.9437797, 0.4535961, 0.9401076, 0.9939561, 0.9993951, 0.9999395, 0.999994] + } + }, + "sinCacheT": { + "dtype": "float32", + "shape": [12, 8], + "data": { + "kind": "values", + "values": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.841471, 0.3109836, 0.0998334, 0.0316175, 0.0099998, 0.0031623, 0.001, 0.0003162, 0.9092974, 0.5911271, 0.1986693, 0.0632034, 0.0199987, 0.0063245, 0.002, 0.0006325, 0.14112, 0.8126489, 0.2955202, 0.0947261, 0.0299955, 0.0094867, 0.003, 0.0009487, -0.7568025, 0.9535807, 0.3894183, 0.1261541, 0.0399893, 0.0126488, 0.004, 0.0012649, -0.9589243, 0.9999465, 0.4794255, 0.1574559, 0.0499792, 0.0158107, 0.005, 0.0015811, -0.2794155, 0.9471482, 0.5646425, 0.1886003, 0.059964, 0.0189725, 0.006, 0.0018974, 0.6569866, 0.8004216, 0.6442177, 0.2195561, 0.0699428, 0.0221341, 0.0069999, 0.0022136, 0.9893582, 0.5743178, 0.7173561, 0.2502924, 0.0799147, 0.0252955, 0.0079999, 0.0025298, 0.4121185, 0.2912591, 0.7833269, 0.2807784, 0.0898785, 0.0284567, 0.0089999, 0.002846, -0.5440211, -0.0206835, 0.841471, 0.3109836, 0.0998334, 0.0316175, 0.0099998, 0.0031623, -0.9999902, -0.330575, 0.8912074, 0.3408779, 0.1097783, 0.034778, 0.0109998, 0.0034785] + } + }, + "qNormWeightT": { + "dtype": "float32", + "shape": [16], + "data": { + "kind": "values", + "values": [0.8, 0.81, 0.82, 0.83, 0.84, 0.85, 0.86, 0.87, 0.88, 0.89, 0.9, 0.91, 0.92, 0.93, 0.94, 0.95] + } + }, + "kNormWeightT": { + "dtype": "float32", + "shape": [16], + "data": { + "kind": "values", + "values": [0.9, 0.908, 0.916, 0.924, 0.932, 0.94, 0.948, 0.956, 0.964, 0.972, 0.98, 0.988, 0.996, 1.004, 1.012, 1.02] + } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 32], "tolerance": 0.001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 4, 16], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 4, 16], "tolerance": 0.0001 } + } + }, + { + "name": "ort_qknorm_rotary_prefill_b1q3p0_h2kv1d16", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "do_rotary": 1, "qk_norm_epsilon": 0.00001 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 3, 32], + "data": { + "kind": "values", + "values": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12, 0.14, 0.16, 0.18, 0.2, 0.22, 0.24, 0.26, 0.28, 0.3, 0.32, 0.34, 0.36, 0.38, 0.4, 0.42, 0.44, 0.46, 0.48, 0.5, 0.52, 0.54, 0.56, 0.58, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7, 0.72, 0.74, 0.76, 0.78, 0.8, 0.82, 0.84, 0.86, 0.88, 0.9, 0.92, 0.94, 0.96, 0.98, 1.0, 1.02, 1.04, 1.06, 1.08, 1.1, 1.12, 1.14, 1.16, 1.18, 1.2, 1.22, 1.24, 1.26, 1.28, 1.3, 1.32, 1.34, 1.36, 1.38, 1.4, 1.42, 1.44, 1.46, 1.48, 1.5, 1.52, 1.54, 1.56, 1.58, 1.6, 1.62, 1.64, 1.66, 1.68, 1.7, 1.72, 1.74, 1.76, 1.78, 1.8, 1.82, 1.84, 1.86, 1.88, 1.9, 1.92] + } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 3, 16], + "data": { + "kind": "values", + "values": [0.051, 0.068, 0.085, 0.102, 0.119, 0.136, 0.153, 0.17, 0.187, 0.204, 0.221, 0.238, 0.255, 0.272, 0.289, 0.306, 0.323, 0.34, 0.357, 0.374, 0.391, 0.408, 0.425, 0.442, 0.459, 0.476, 0.493, 0.51, 0.527, 0.544, 0.561, 0.578, 0.595, 0.612, 0.629, 0.646, 0.663, 0.68, 0.697, 0.714, 0.731, 0.748, 0.765, 0.782, 0.799, 0.816, 0.833, 0.85] + } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 3, 16], + "data": { + "kind": "values", + "values": [0.013, 0.026, 0.039, 0.052, 0.065, 0.078, 0.091, 0.104, 0.117, 0.013, 0.026, 0.039, 0.052, 0.065, 0.078, 0.091, 0.104, 0.117, 0.013, 0.026, 0.039, 0.052, 0.065, 0.078, 0.091, 0.104, 0.117, 0.013, 0.026, 0.039, 0.052, 0.065, 0.078, 0.091, 0.104, 0.117, 0.013, 0.026, 0.039, 0.052, 0.065, 0.078, 0.091, 0.104, 0.117, 0.013, 0.026, 0.039] + } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [2] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [3] } }, + "cosCacheT": { + "dtype": "float32", + "shape": [11, 8], + "data": { + "kind": "values", + "values": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.5403023, 0.9504153, 0.9950042, 0.9995, 0.99995, 0.999995, 0.9999995, 1.0, -0.4161468, 0.8065784, 0.9800666, 0.9980007, 0.9998, 0.99998, 0.999998, 0.9999998, -0.9899925, 0.5827536, 0.9553365, 0.9955034, 0.99955, 0.999955, 0.9999955, 0.9999996, -0.6536436, 0.3011375, 0.921061, 0.9920107, 0.9992001, 0.99992, 0.999992, 0.9999992, 0.2836622, -0.0103423, 0.8775826, 0.987526, 0.9987503, 0.999875, 0.9999875, 0.9999988, 0.9601703, -0.3207965, 0.8253356, 0.9820539, 0.9982005, 0.99982, 0.999982, 0.9999982, 0.7539023, -0.5994374, 0.7648422, 0.9755999, 0.997551, 0.999755, 0.9999755, 0.9999976, -0.1455, -0.8186325, 0.6967067, 0.9681703, 0.9968017, 0.99968, 0.999968, 0.9999968, -0.9111303, -0.9566442, 0.62161, 0.9597726, 0.9959527, 0.999595, 0.9999595, 0.999996, -0.8390715, -0.9997861, 0.5403023, 0.9504153, 0.9950042, 0.9995, 0.99995, 0.999995] + } + }, + "sinCacheT": { + "dtype": "float32", + "shape": [11, 8], + "data": { + "kind": "values", + "values": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.841471, 0.3109836, 0.0998334, 0.0316175, 0.0099998, 0.0031623, 0.001, 0.0003162, 0.9092974, 0.5911271, 0.1986693, 0.0632034, 0.0199987, 0.0063245, 0.002, 0.0006325, 0.14112, 0.8126489, 0.2955202, 0.0947261, 0.0299955, 0.0094867, 0.003, 0.0009487, -0.7568025, 0.9535807, 0.3894183, 0.1261541, 0.0399893, 0.0126488, 0.004, 0.0012649, -0.9589243, 0.9999465, 0.4794255, 0.1574559, 0.0499792, 0.0158107, 0.005, 0.0015811, -0.2794155, 0.9471482, 0.5646425, 0.1886003, 0.059964, 0.0189725, 0.006, 0.0018974, 0.6569866, 0.8004216, 0.6442177, 0.2195561, 0.0699428, 0.0221341, 0.0069999, 0.0022136, 0.9893582, 0.5743178, 0.7173561, 0.2502924, 0.0799147, 0.0252955, 0.0079999, 0.0025298, 0.4121185, 0.2912591, 0.7833269, 0.2807784, 0.0898785, 0.0284567, 0.0089999, 0.002846, -0.5440211, -0.0206835, 0.841471, 0.3109836, 0.0998334, 0.0316175, 0.0099998, 0.0031623] + } + }, + "qNormWeightT": { + "dtype": "float32", + "shape": [16], + "data": { + "kind": "values", + "values": [0.85, 0.855, 0.86, 0.865, 0.87, 0.875, 0.88, 0.885, 0.89, 0.895, 0.9, 0.905, 0.91, 0.915, 0.92, 0.925] + } + }, + "kNormWeightT": { + "dtype": "float32", + "shape": [16], + "data": { + "kind": "values", + "values": [0.92, 0.924, 0.928, 0.932, 0.936, 0.94, 0.944, 0.948, 0.952, 0.956, 0.96, 0.964, 0.968, 0.972, 0.976, 0.98] + } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 3, 32], "tolerance": 0.001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 3, 16], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 3, 16], "tolerance": 0.0001 } + } + }, + { + "name": "ort_sharedkv_slidingwindow_b1q4p32_h2kv1d8_w16", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "local_window_size": 16 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 4, 16], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_sharedkv_rotary_multibatch_b2q1p8_h2kv1d16_input_queryT" } + } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 8], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 8], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 32, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_large_past_key_t_pattern" } } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 32, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_large_past_value_t_pattern" } } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 4, 16], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 32, 8], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 32, 8], "tolerance": 0.0001 } + } + }, + { + "name": "ort_quant_int8_pertensor_b1q4_h2kv1d8", + "attrs": { + "num_heads": 2, + "kv_num_heads": 1, + "kv_cache_bit_width": 8, + "k_quant_type": "PER_TENSOR", + "v_quant_type": "PER_TENSOR" + }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 4, 16], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_quant_query_t" } } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 4, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_quant_key_t" } } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 4, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_quant_value_t" } } + }, + "pastKeyT": { + "dtype": "int8", + "shape": [1, 1, 4, 8], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_quant_int8_pertensor_b1q4_h2kv1d8_input_pastKeyT" } + } + }, + "pastValueT": { + "dtype": "int8", + "shape": [1, 1, 4, 8], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_quant_int8_pertensor_b1q4_h2kv1d8_input_pastKeyT" } + } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [3] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } }, + "kScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [0.0039134] } }, + "vScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [0.0038802] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 4, 16], "tolerance": 0.05 }, + "presentKeyT": { "dtype": "int8", "shape": [1, 1, 4, 8], "tolerance": 0 }, + "presentValueT": { "dtype": "int8", "shape": [1, 1, 4, 8], "tolerance": 0 } + } + }, + { + "name": "ort_quant_int8_perchannel_b1q4_h2kv1d8", + "attrs": { + "num_heads": 2, + "kv_num_heads": 1, + "kv_cache_bit_width": 8, + "k_quant_type": "PER_CHANNEL", + "v_quant_type": "PER_CHANNEL" + }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 4, 16], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_quant_query_t" } } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 4, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_quant_key_t" } } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 4, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_quant_value_t" } } + }, + "pastKeyT": { + "dtype": "int8", + "shape": [1, 1, 4, 8], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_quant_int8_pertensor_b1q4_h2kv1d8_input_pastKeyT" } + } + }, + "pastValueT": { + "dtype": "int8", + "shape": [1, 1, 4, 8], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_quant_int8_pertensor_b1q4_h2kv1d8_input_pastKeyT" } + } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [3] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } }, + "kScaleT": { + "dtype": "float32", + "shape": [8], + "data": { + "kind": "values", + "values": [0.0039134, 0.002415, 0.0023732, 0.0021473, 0.0025679, 0.0025398, 0.0034738, 0.0036063] + } + }, + "vScaleT": { + "dtype": "float32", + "shape": [8], + "data": { + "kind": "values", + "values": [0.0020182, 0.0017471, 0.0036659, 0.0021576, 0.0038802, 0.0031283, 0.0037608, 0.0038594] + } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 4, 16], "tolerance": 0.05 }, + "presentKeyT": { "dtype": "int8", "shape": [1, 1, 4, 8], "tolerance": 0 }, + "presentValueT": { "dtype": "int8", "shape": [1, 1, 4, 8], "tolerance": 0 } + } + }, + { + "name": "ort_quant_int4_biased_nibbles_exact_b1q1_h1kv1d8", + "provenance": { + "source": "ONNX Runtime MLAS quantized-KV ABI (onnxruntime/core/mlas/inc/mlas_qkv_quant.h)", + "notes": "Pins every signed INT4 region at byte level. ORT stores q in [-8, 7] as q + 8, with the even head coordinate in the low nibble; a one-token attention row also pins dequantization independently through the float output." + }, + "attrs": { + "num_heads": 1, + "kv_num_heads": 1, + "kv_cache_bit_width": 4, + "k_quant_type": "PER_TENSOR", + "v_quant_type": "PER_TENSOR" + }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [1, 1, 8], "data": { "kind": "constant", "value": 0.0 } }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 8], + "data": { "kind": "values", "values": [-8.0, -7.0, -1.0, 0.0, 1.0, 6.0, 7.0, -2.0] } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 8], + "data": { "kind": "values", "values": [7.0, 6.0, 1.0, 0.0, -1.0, -7.0, -8.0, 2.0] } + }, + "pastKeyT": { "dtype": "uint8", "shape": [1, 1, 1, 4], "data": { "kind": "constant", "value": 0 } }, + "pastValueT": { "dtype": "uint8", "shape": [1, 1, 1, 4], "data": { "kind": "constant", "value": 0 } }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [0] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [1] } }, + "kScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [1.0] } }, + "vScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [1.0] } } + }, + "outputs": { + "outputT": { + "dtype": "float32", + "shape": [1, 1, 8], + "data": { "kind": "values", "values": [7.0, 6.0, 1.0, 0.0, -1.0, -7.0, -8.0, 2.0] }, + "tolerance": 0 + }, + "presentKeyT": { + "dtype": "uint8", + "shape": [1, 1, 1, 4], + "data": { "kind": "values", "values": [16, 135, 233, 111] }, + "tolerance": 0 + }, + "presentValueT": { + "dtype": "uint8", + "shape": [1, 1, 1, 4], + "data": { "kind": "values", "values": [239, 137, 23, 160] }, + "tolerance": 0 + } + } + }, + { + "name": "ort_quant_int4_pertensor_b1q4_h2kv1d8", + "attrs": { + "num_heads": 2, + "kv_num_heads": 1, + "kv_cache_bit_width": 4, + "k_quant_type": "PER_TENSOR", + "v_quant_type": "PER_TENSOR" + }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 4, 16], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_quant_query_t" } } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 4, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_quant_key_t" } } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 4, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_quant_value_t" } } + }, + "pastKeyT": { + "dtype": "uint8", + "shape": [1, 1, 4, 4], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_quant_int4_pertensor_b1q4_h2kv1d8_input_pastKeyT" } + } + }, + "pastValueT": { + "dtype": "uint8", + "shape": [1, 1, 4, 4], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_quant_int4_pertensor_b1q4_h2kv1d8_input_pastKeyT" } + } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [3] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } }, + "kScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [0.0710003] } }, + "vScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [0.0703979] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 4, 16], "tolerance": 0.15 }, + "presentKeyT": { "dtype": "uint8", "shape": [1, 1, 4, 4], "tolerance": 0 }, + "presentValueT": { "dtype": "uint8", "shape": [1, 1, 4, 4], "tolerance": 0 } + } + }, + { + "name": "ort_quant_int4_perchannel_b1q4_h2kv1d8", + "attrs": { + "num_heads": 2, + "kv_num_heads": 1, + "kv_cache_bit_width": 4, + "k_quant_type": "PER_CHANNEL", + "v_quant_type": "PER_CHANNEL" + }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 4, 16], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_quant_query_t" } } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 4, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_quant_key_t" } } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 4, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_quant_value_t" } } + }, + "pastKeyT": { + "dtype": "uint8", + "shape": [1, 1, 4, 4], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_quant_int4_pertensor_b1q4_h2kv1d8_input_pastKeyT" } + } + }, + "pastValueT": { + "dtype": "uint8", + "shape": [1, 1, 4, 4], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_quant_int4_pertensor_b1q4_h2kv1d8_input_pastKeyT" } + } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [3] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } }, + "kScaleT": { + "dtype": "float32", + "shape": [8], + "data": { + "kind": "values", + "values": [0.0710003, 0.0438154, 0.0430563, 0.038959, 0.0465887, 0.0460787, 0.063024, 0.0654294] + } + }, + "vScaleT": { + "dtype": "float32", + "shape": [8], + "data": { + "kind": "values", + "values": [0.0366163, 0.0316981, 0.0665106, 0.0391457, 0.0703979, 0.0567561, 0.0682314, 0.0700206] + } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 4, 16], "tolerance": 0.15 }, + "presentKeyT": { "dtype": "uint8", "shape": [1, 1, 4, 4], "tolerance": 0 }, + "presentValueT": { "dtype": "uint8", "shape": [1, 1, 4, 4], "tolerance": 0 } + } + }, + { + "name": "ort_quant_int8_multibatch_b2q4_h4kv2d16", + "attrs": { + "num_heads": 4, + "kv_num_heads": 2, + "kv_cache_bit_width": 8, + "k_quant_type": "PER_TENSOR", + "v_quant_type": "PER_TENSOR" + }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [2, 4, 64], + "data": { + "kind": "values", + "values": [0.082308, 0.019819, -0.034024, 0.277037, -0.077135, -0.466628, -0.082611, 0.308729, 0.11234, 0.214904, -0.317721, 0.016195, 0.061967, -0.251316, 0.419605, -0.389136, 0.158377, -0.171533, -0.314711, 0.171006, 0.418748, -0.368512, -0.089387, 0.349135, -0.090375, -0.467301, 0.066158, 0.304013, -0.351828, 0.402985, -0.01996, -0.048985, 0.204581, 0.335118, 0.279197, 0.124608, -0.275582, 0.141191, 0.422468, -0.253317, -0.410373, 0.46529, -0.205779, -0.405497, 0.436971, -0.14485, 0.327278, -0.238, -0.247373, -0.363456, -0.37611, 0.408198, -0.238281, -0.065844, 0.043498, -0.113526, -0.168485, -0.317308, -0.273944, -0.151551, 0.107045, -0.028029, 0.01964, 0.416226, -0.402106, -0.43025, 0.281105, 0.01674, 0.434919, -0.033959, -0.481453, -0.028033, -0.484372, -0.217347, -0.155699, 0.438746, 0.256996, -0.108279, 0.20724, -0.208042, -0.325565, 0.086976, 0.373127, 0.044809, -0.494295, -0.348512, -0.378245, 0.399983, -0.40152, -0.334346, 0.129636, -0.474602, 0.280348, 0.399162, -0.493008, -0.35168, -0.070028, -0.369303, 0.457786, 0.061587, 0.468188, 0.239124, 0.031557, 0.135702, -0.012245, -0.090088, 0.435791, 0.430518, 0.240547, 0.300536, -0.122395, 0.000296, 0.200901, -0.491379, 0.40346, 0.485869, 0.423899, 0.18656, 0.205686, 0.087662, 0.417308, 0.146378, 0.364543, 0.013327, -0.254039, -0.201427, 0.494586, 0.49373, 0.08729, 0.268691, 0.264073, -0.038945, -0.043208, -0.46461, 0.256173, 0.482762, 0.322169, 0.433737, 0.140714, 0.045159, -0.175119, -0.132516, -0.374761, -0.219674, -0.172124, 0.074085, 0.154098, -0.269842, 0.008243, 0.286197, -0.175656, 0.241259, -0.286464, -0.36659, 0.487174, -0.415547, -0.199484, -0.372034, -0.496337, -0.107356, 0.17507, 0.078215, 0.123551, 0.010797, -0.24795, -0.226458, -0.044176, 0.35177, 0.053783, -0.215451, 0.49984, 0.351761, 0.166002, -0.277798, 0.205756, -0.380893, 0.269957, -0.200274, 0.032998, 0.370204, 0.469432, 0.040976, -0.024468, 0.246272, 0.131451, 0.479044, -0.477325, 0.104742, 0.459394, -0.311143, -0.475788, -0.121004, -0.48271, 0.497501, 0.428552, 0.100217, 0.424031, 0.21603, 0.362687, 0.175099, -0.063742, 0.236517, 0.022691, 0.013166, 0.138668, 0.338081, -0.156478, 0.097452, 0.460495, -0.454347, 0.416872, -0.407713, -0.263091, -0.114778, -0.374197, -0.418052, -0.46591, -0.120674, 0.464943, 0.033911, 0.456725, 0.126508, 0.014974, -0.419527, -0.343328, -0.191469, -0.459838, -0.287529, 0.261552, 0.469055, 0.17769, 0.478872, -0.057993, -0.092439, 0.187538, 0.285966, 0.336384, -0.467742, 0.231653, -0.110081, -0.44841, 0.015721, -0.348823, 0.176437, 0.458867, -0.402987, 0.210459, -0.178714, 0.00324, -0.12595, 0.276625, 0.425428, 0.24539, 0.429934, -0.09656, 0.218509, 0.354692, 0.137039, -0.211786, -0.075539, 0.272182, 0.482569, -0.429488, -0.117038, 0.262192, -0.240649, 0.441403, 0.020503, 0.071406, 0.465623, -0.467987, -0.23518, -0.085909, -0.242461, 0.050122, 0.496432, 0.002249, 0.298175, -0.366448, -0.368967, -0.160562, 0.346761, -0.110883, -0.073492, -0.00298, 0.431897, -0.414151, 0.10829, -0.408426, 0.452506, 0.471292, 0.406843, 0.18224, -0.449728, 0.179579, 0.100992, 0.306726, -0.402692, 0.200057, -0.339132, 0.37474, -0.357187, 0.375219, -0.415784, 0.137598, 0.162463, -0.103334, -0.480198, 0.45275, 0.454283, 0.267887, -0.228311, -0.438028, 0.024755, 0.202564, -0.221881, -0.398004, 0.308912, -0.072809, -0.341247, -0.433707, -0.431835, -0.143269, 0.424021, 0.436794, 0.309616, 0.112069, -0.059665, -0.295731, 0.0706, -0.151977, 0.244904, -0.003859, -0.087782, -0.139366, 0.303916, 0.030461, 0.465602, 0.442519, -0.192174, -0.385189, -0.125336, -0.044985, 0.0975, -0.091122, 0.479075, 0.085865, -0.185023, -0.330551, -0.418759, 0.261208, 0.372581, -0.303426, 0.140999, 0.180613, -0.165851, 0.199661, -0.258198, -0.056089, 0.302114, 0.302235, -0.327852, -0.26143, 0.404201, 0.404115, 0.465976, -0.443626, -0.026518, -0.380222, -0.254187, -0.09186, -0.484467, 0.011627, -0.287607, 0.00524, -0.443535, -0.163056, 0.124518, 0.376665, 0.46555, -0.472741, 0.1129, 0.390388, -0.153788, -0.38111, -0.328261, -0.352226, 0.089131, 0.353412, 0.171782, -0.280219, 0.181202, 0.166785, -0.261378, 0.193722, 0.364103, -0.214081, -0.160024, 0.499305, -0.263777, 0.1514, -0.074297, 0.482737, 0.391704, -0.114859, 0.464298, 0.197613, 0.190332, -0.478293, 0.219628, -0.118639, -0.247947, 0.222058, 0.111651, -0.414027, -0.230171, 0.188468, -0.495132, 0.431796, 0.234622, 0.499363, 0.268255, -0.046892, -0.333777, -0.417169, 0.450441, -0.495751, -0.0563, 0.047558, -0.086143, 0.195041, -0.170799, 0.089926, -0.22837, 0.324269, -0.221134, 0.196046, -0.029047, -0.220062, 0.1346, 0.357453, -0.133301, 0.375099, 0.290454, -0.480774, 0.129009, -0.412047, -0.07263, 0.47928, -0.171381, -0.324985, -0.089974, -0.042417, 0.043675, -0.37393, -0.060451, 0.054037, -0.168224, -0.070328, -0.444165, -0.186212, -0.088993, -0.382051, -0.416774, 0.118706, -0.1063, -0.440555, -0.3867, -0.376214, 0.416686, 0.468768, -0.381485, -0.04278, 0.011778, -0.398224, 0.29047, -0.097106, 0.060236, -0.394933, 0.196519, -0.215948, -0.325027, 0.488514, 0.348497, -0.233109, -0.145759, 0.189593, 0.277303, 0.302813, -0.249453, -0.301736, 0.207099, 0.428114, 0.398565, 0.211142, 0.422859, 0.139839, -0.254228, 0.283725, 0.425034, -0.433446, 0.329573, 0.449404, -0.034924, 0.152795, 0.444658, 0.274038, -0.444432, -0.424057, 0.144347, 0.314806, -0.451287] + } + }, + "keyT": { + "dtype": "float32", + "shape": [2, 4, 32], + "data": { + "kind": "values", + "values": [0.372574, -0.306708, 0.184696, 0.272713, -0.148147, -0.122402, 0.033584, -0.125638, 0.227005, -0.30606, 0.136125, 0.167737, 0.018802, -0.24656, -0.205723, 0.118218, 0.497002, 0.044897, -0.050453, -0.078412, -0.326121, 0.042089, 0.441168, 0.458006, -0.149514, 0.059117, 0.301394, -0.209086, 0.059405, 0.322551, 0.181637, -0.070391, 0.247732, -0.27088, -0.496768, -0.425765, -0.455956, -0.242304, 0.426551, 0.061955, 0.453098, -0.495666, 0.232227, 0.233034, 0.322021, 0.309097, 0.208241, 0.435606, -0.078269, -0.05656, -0.363385, 0.387851, 0.487342, -0.173121, -0.026534, 0.116354, 0.061299, 0.241717, -0.183831, -0.310622, 0.483011, 0.01456, -0.248619, -0.036185, -0.1313, -0.366616, -0.322331, 0.064553, -0.074749, 0.268896, -0.303493, -0.184979, -0.058216, -0.331057, 0.165371, -0.162349, -0.197279, 0.394721, 0.006702, -0.456985, -0.133926, 0.367525, -0.133525, -0.273619, 0.050734, 0.194798, 0.216127, -0.009556, -0.02352, 0.244232, -0.008405, -0.287023, -0.499521, -0.065344, -0.136786, -0.221845, 0.494022, 0.212482, -0.269775, -0.209704, 0.203238, -0.317563, -0.466362, -0.011215, 0.037548, 0.361352, -0.018789, -0.417094, 0.129602, -0.476261, 0.278283, -0.274642, -0.489551, -0.392078, 0.458218, -0.156817, 0.243694, -0.34854, 0.406212, -0.455124, 0.362385, 0.224619, 0.1702, -0.177594, -0.047613, 0.330641, -0.347365, -0.233084, 0.225994, 0.241559, 0.276866, -0.483153, -0.070481, -0.26169, -0.278121, -0.342778, -0.07138, -0.276106, 0.319151, -0.211132, 0.315116, -0.321671, 0.205356, 0.486297, -0.120972, -0.075846, -0.175313, 0.363014, 0.289611, 0.17123, -0.125469, 0.462999, -0.170879, 0.309588, 0.288264, -0.302894, 0.023059, -0.080933, 0.348892, -0.456864, 0.01066, 0.464513, -0.472691, -0.271665, -0.000643, -0.354744, 0.138917, 0.113056, -0.353016, -0.132346, 0.412347, 0.494355, -0.284535, 0.309418, 0.064046, -0.451756, -0.16027, -0.455509, -0.027529, -0.370621, -0.244376, 0.197224, 0.045646, -0.353087, -0.169454, -0.405403, -0.124184, -0.264728, 0.240572, -0.114761, 0.274982, -0.061399, -0.362185, -0.406011, 0.285021, -0.372359, -0.348234, 0.080765, -0.243814, -0.132237, 0.068372, -0.127534, 0.087701, 0.456935, 0.030599, 0.336685, 0.224224, -0.147638, 0.104225, -0.334044, -0.498085, -0.295465, -0.447379, -0.358693, 0.437865, 0.217468, -0.335733, 0.143857, 0.056636, 0.053846, 0.476752, -0.116792, -0.152346, 0.003849, -0.258996, 0.311385, -0.052781, 0.136397, 0.46951, 0.191076, -0.361128, -0.348434, -0.087733, -0.213087, -0.234132, 0.152927, 0.30422, 0.148556, 0.149514, -0.441438, 0.227932, 0.454319, -0.392891, 0.369486, -0.06476, -0.115906, -0.336748, -0.485737, 0.471642, -0.009675, -0.451748, 0.488523, -0.052824, 0.035777, -0.422595, 0.008167] + } + }, + "valueT": { + "dtype": "float32", + "shape": [2, 4, 32], + "data": { + "kind": "values", + "values": [0.097056, -0.200735, -0.168325, 0.191973, 0.492785, -0.397293, 0.47762, 0.470182, -0.256314, -0.16696, -0.157802, 0.27402, 0.404261, 0.289624, -0.477124, -0.490144, -0.067094, 0.221887, 0.084429, -0.272333, -0.492298, 0.063575, -0.344991, 0.405092, 0.051202, -0.053699, 0.465574, 0.05767, 0.307941, -0.038854, -0.073572, -0.106016, 0.318435, -0.362953, 0.334614, 0.234486, 0.133419, -0.395259, -0.194075, 0.412745, 0.112058, -0.085, -0.474731, 0.377861, -0.352095, -0.020081, -0.090857, -0.283196, -0.333665, -0.394006, -0.226029, 0.315428, -0.262786, 0.087747, 0.104044, 0.376898, 0.14656, -0.169341, -0.487171, 0.0163, -0.229075, 0.472135, -0.008622, 0.150163, 0.098277, -0.030727, 0.037601, -0.100338, 0.239101, -0.377817, -0.239849, 0.226232, 0.271498, -0.498403, 0.050661, 0.018367, -0.235993, 0.249528, 0.418097, 0.348326, -0.18008, -0.278273, -0.421034, 0.117528, 0.486828, 0.440684, 0.39714, -0.190651, -0.124853, 0.32793, 0.26703, -0.431887, -0.17061, -0.315615, -0.466127, 0.322982, 0.495907, -0.112505, 0.044886, 0.175538, -0.369124, -0.226068, 0.425015, -0.230226, -0.007732, 0.176571, 0.17106, -0.136738, -0.340452, -0.415466, 0.15789, -0.18176, -0.375137, 0.395014, -0.349461, 0.231378, 0.499463, -0.274344, 0.415889, 0.420953, -0.320896, 0.25623, 0.027736, -0.009791, -0.187809, 0.068355, -0.041058, 0.185634, -0.085599, -0.047983, -0.445536, 0.268902, -0.026461, 0.147633, 0.478985, -0.378748, 0.388383, -0.007587, -0.267099, -0.200979, 0.085262, 0.439772, -0.169571, 0.259115, -0.343885, -0.079466, 0.233561, -0.02686, 0.361789, -0.485315, -0.364367, -0.351037, 0.344318, -0.097577, 0.188259, -0.221562, -0.108062, 0.063255, 0.265364, 0.230061, 0.100586, 0.191891, -0.141915, -0.394709, -0.224366, -0.200318, 0.194275, 0.077297, -0.082394, 0.470361, -0.323629, -0.06913, -0.364369, 0.238824, -0.268632, -0.477731, -0.167623, 0.061713, -0.333345, -0.122272, -0.495773, 0.348542, 0.125462, -0.350613, 0.000428, 0.022125, 0.235659, -0.352601, 0.084987, 0.077044, 0.045669, -0.333042, 0.145038, 0.10492, 0.341247, -0.171008, 0.389457, -0.244772, 0.436854, 0.153288, 0.381451, -0.399592, 0.035404, -0.417219, -0.251114, 0.374026, -0.249891, 0.17777, 0.4161, 0.05073, 0.382196, -0.301198, 0.200946, -0.26325, -0.45178, 0.296979, 0.120824, -0.160299, -0.249254, -0.469557, 0.251445, 0.48643, 0.486136, 0.425768, 0.482082, -0.342344, 0.483198, -0.196957, 0.011052, 0.008165, -0.333561, -0.416641, -0.074819, -0.158708, -0.062314, 0.297481, -0.292988, 0.206461, 0.369809, -0.485563, 0.013486, -0.411489, -0.093694, 0.408774, -0.430134, 0.157578, 0.160498, 0.185247, -0.477358, 0.048266, 0.408082, -0.420333, 0.09517, -0.363128, 0.366795, 0.154684] + } + }, + "pastKeyT": { + "dtype": "int8", + "shape": [2, 2, 4, 16], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_quant_int8_multibatch_b2q4_h4kv2d16_input_pastKeyT" } + } + }, + "pastValueT": { + "dtype": "int8", + "shape": [2, 2, 4, 16], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_quant_int8_multibatch_b2q4_h4kv2d16_input_pastKeyT" } + } + }, + "seqlensKT": { "dtype": "int32", "shape": [2], "data": { "kind": "values", "values": [3, 3] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } }, + "kScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [0.0039332] } }, + "vScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [0.0039328] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [2, 4, 64], "tolerance": 0.05 }, + "presentKeyT": { "dtype": "int8", "shape": [2, 2, 4, 16], "tolerance": 0 }, + "presentValueT": { "dtype": "int8", "shape": [2, 2, 4, 16], "tolerance": 0 } + } + }, + { + "name": "ort_quant_int8_largehead_b1q8_h2kv1d64", + "attrs": { + "num_heads": 2, + "kv_num_heads": 1, + "kv_cache_bit_width": 8, + "k_quant_type": "PER_TENSOR", + "v_quant_type": "PER_TENSOR" + }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 8, 128], + "data": { + "kind": "values", + "values": [0.082308, 0.019819, -0.034024, 0.277037, -0.077135, -0.466628, -0.082611, 0.308729, 0.11234, 0.214904, -0.317721, 0.016195, 0.061967, -0.251316, 0.419605, -0.389136, 0.158377, -0.171533, -0.314711, 0.171006, 0.418748, -0.368512, -0.089387, 0.349135, -0.090375, -0.467301, 0.066158, 0.304013, -0.351828, 0.402985, -0.01996, -0.048985, 0.204581, 0.335118, 0.279197, 0.124608, -0.275582, 0.141191, 0.422468, -0.253317, -0.410373, 0.46529, -0.205779, -0.405497, 0.436971, -0.14485, 0.327278, -0.238, -0.247373, -0.363456, -0.37611, 0.408198, -0.238281, -0.065844, 0.043498, -0.113526, -0.168485, -0.317308, -0.273944, -0.151551, 0.107045, -0.028029, 0.01964, 0.416226, -0.402106, -0.43025, 0.281105, 0.01674, 0.434919, -0.033959, -0.481453, -0.028033, -0.484372, -0.217347, -0.155699, 0.438746, 0.256996, -0.108279, 0.20724, -0.208042, -0.325565, 0.086976, 0.373127, 0.044809, -0.494295, -0.348512, -0.378245, 0.399983, -0.40152, -0.334346, 0.129636, -0.474602, 0.280348, 0.399162, -0.493008, -0.35168, -0.070028, -0.369303, 0.457786, 0.061587, 0.468188, 0.239124, 0.031557, 0.135702, -0.012245, -0.090088, 0.435791, 0.430518, 0.240547, 0.300536, -0.122395, 0.000296, 0.200901, -0.491379, 0.40346, 0.485869, 0.423899, 0.18656, 0.205686, 0.087662, 0.417308, 0.146378, 0.364543, 0.013327, -0.254039, -0.201427, 0.494586, 0.49373, 0.08729, 0.268691, 0.264073, -0.038945, -0.043208, -0.46461, 0.256173, 0.482762, 0.322169, 0.433737, 0.140714, 0.045159, -0.175119, -0.132516, -0.374761, -0.219674, -0.172124, 0.074085, 0.154098, -0.269842, 0.008243, 0.286197, -0.175656, 0.241259, -0.286464, -0.36659, 0.487174, -0.415547, -0.199484, -0.372034, -0.496337, -0.107356, 0.17507, 0.078215, 0.123551, 0.010797, -0.24795, -0.226458, -0.044176, 0.35177, 0.053783, -0.215451, 0.49984, 0.351761, 0.166002, -0.277798, 0.205756, -0.380893, 0.269957, -0.200274, 0.032998, 0.370204, 0.469432, 0.040976, -0.024468, 0.246272, 0.131451, 0.479044, -0.477325, 0.104742, 0.459394, -0.311143, -0.475788, -0.121004, -0.48271, 0.497501, 0.428552, 0.100217, 0.424031, 0.21603, 0.362687, 0.175099, -0.063742, 0.236517, 0.022691, 0.013166, 0.138668, 0.338081, -0.156478, 0.097452, 0.460495, -0.454347, 0.416872, -0.407713, -0.263091, -0.114778, -0.374197, -0.418052, -0.46591, -0.120674, 0.464943, 0.033911, 0.456725, 0.126508, 0.014974, -0.419527, -0.343328, -0.191469, -0.459838, -0.287529, 0.261552, 0.469055, 0.17769, 0.478872, -0.057993, -0.092439, 0.187538, 0.285966, 0.336384, -0.467742, 0.231653, -0.110081, -0.44841, 0.015721, -0.348823, 0.176437, 0.458867, -0.402987, 0.210459, -0.178714, 0.00324, -0.12595, 0.276625, 0.425428, 0.24539, 0.429934, -0.09656, 0.218509, 0.354692, 0.137039, -0.211786, -0.075539, 0.272182, 0.482569, -0.429488, -0.117038, 0.262192, -0.240649, 0.441403, 0.020503, 0.071406, 0.465623, -0.467987, -0.23518, -0.085909, -0.242461, 0.050122, 0.496432, 0.002249, 0.298175, -0.366448, -0.368967, -0.160562, 0.346761, -0.110883, -0.073492, -0.00298, 0.431897, -0.414151, 0.10829, -0.408426, 0.452506, 0.471292, 0.406843, 0.18224, -0.449728, 0.179579, 0.100992, 0.306726, -0.402692, 0.200057, -0.339132, 0.37474, -0.357187, 0.375219, -0.415784, 0.137598, 0.162463, -0.103334, -0.480198, 0.45275, 0.454283, 0.267887, -0.228311, -0.438028, 0.024755, 0.202564, -0.221881, -0.398004, 0.308912, -0.072809, -0.341247, -0.433707, -0.431835, -0.143269, 0.424021, 0.436794, 0.309616, 0.112069, -0.059665, -0.295731, 0.0706, -0.151977, 0.244904, -0.003859, -0.087782, -0.139366, 0.303916, 0.030461, 0.465602, 0.442519, -0.192174, -0.385189, -0.125336, -0.044985, 0.0975, -0.091122, 0.479075, 0.085865, -0.185023, -0.330551, -0.418759, 0.261208, 0.372581, -0.303426, 0.140999, 0.180613, -0.165851, 0.199661, -0.258198, -0.056089, 0.302114, 0.302235, -0.327852, -0.26143, 0.404201, 0.404115, 0.465976, -0.443626, -0.026518, -0.380222, -0.254187, -0.09186, -0.484467, 0.011627, -0.287607, 0.00524, -0.443535, -0.163056, 0.124518, 0.376665, 0.46555, -0.472741, 0.1129, 0.390388, -0.153788, -0.38111, -0.328261, -0.352226, 0.089131, 0.353412, 0.171782, -0.280219, 0.181202, 0.166785, -0.261378, 0.193722, 0.364103, -0.214081, -0.160024, 0.499305, -0.263777, 0.1514, -0.074297, 0.482737, 0.391704, -0.114859, 0.464298, 0.197613, 0.190332, -0.478293, 0.219628, -0.118639, -0.247947, 0.222058, 0.111651, -0.414027, -0.230171, 0.188468, -0.495132, 0.431796, 0.234622, 0.499363, 0.268255, -0.046892, -0.333777, -0.417169, 0.450441, -0.495751, -0.0563, 0.047558, -0.086143, 0.195041, -0.170799, 0.089926, -0.22837, 0.324269, -0.221134, 0.196046, -0.029047, -0.220062, 0.1346, 0.357453, -0.133301, 0.375099, 0.290454, -0.480774, 0.129009, -0.412047, -0.07263, 0.47928, -0.171381, -0.324985, -0.089974, -0.042417, 0.043675, -0.37393, -0.060451, 0.054037, -0.168224, -0.070328, -0.444165, -0.186212, -0.088993, -0.382051, -0.416774, 0.118706, -0.1063, -0.440555, -0.3867, -0.376214, 0.416686, 0.468768, -0.381485, -0.04278, 0.011778, -0.398224, 0.29047, -0.097106, 0.060236, -0.394933, 0.196519, -0.215948, -0.325027, 0.488514, 0.348497, -0.233109, -0.145759, 0.189593, 0.277303, 0.302813, -0.249453, -0.301736, 0.207099, 0.428114, 0.398565, 0.211142, 0.422859, 0.139839, -0.254228, 0.283725, 0.425034, -0.433446, 0.329573, 0.449404, -0.034924, 0.152795, 0.444658, 0.274038, -0.444432, -0.424057, 0.144347, 0.314806, -0.451287, -0.385741, -0.080335, -0.047873, 0.376939, -0.315146, 0.399904, -0.436644, 0.049358, 0.270321, -0.081577, 0.007045, 0.497834, -0.047206, 0.287419, -0.113513, -0.133591, 0.274176, -0.374547, -0.438683, 0.337848, 0.141293, -0.418561, 0.375901, -0.408968, -0.005284, -0.201115, 0.073281, 0.482963, 0.460004, -0.035895, -0.136589, 0.049135, -0.299875, 0.282175, 0.114438, 0.416008, 0.187792, -0.275633, 0.135787, -0.280662, 0.116475, -0.109176, 0.021788, -0.377869, 0.168832, -0.167791, 0.024599, -0.474062, -0.138685, -0.3259, 0.42518, 0.069272, 0.183697, 0.387862, 0.37106, 0.00688, -0.247732, -0.448247, -0.350347, 0.063624, 0.195458, 0.446024, -0.134129, -0.143277, -0.009375, 0.325736, -0.007219, 0.480106, 0.470773, 0.215081, -0.235937, -0.155384, 0.219936, 0.066921, -0.122987, 0.367834, -0.424016, -0.368677, 0.474843, 0.107256, 0.459606, -0.123179, -0.122976, -0.069416, 0.248209, 0.308166, 0.026435, -0.183133, -0.08651, 0.024308, -0.37795, -0.117734, 0.141435, -0.139633, 0.405927, 0.224162, -0.346517, 0.457331, -0.42676, 0.177389, 0.447615, -0.396755, -0.225888, 0.076314, 0.178116, -0.310367, -0.324007, 0.116533, 0.392757, -0.373101, -0.444667, 0.409925, -0.15385, 0.062473, -0.112693, -0.173186, -0.391978, 0.245429, 0.402328, -0.281006, 0.185169, 0.381345, -0.421882, 0.117357, 0.375817, -0.178851, 0.036069, 0.497887, 0.075174, -0.072308, -0.189351, -0.290239, -0.249114, -0.036188, -0.039388, 0.159305, -0.192823, -0.15437, 0.490748, -0.15643, -0.443594, 0.104827, 0.052936, 0.327253, 0.016794, -0.096336, 0.395238, 0.256741, 0.255081, 0.446476, 0.027791, -0.494994, 0.492841, -0.368592, 0.074254, -0.458887, -0.304766, 0.286132, -0.222463, 0.118084, -0.139078, -0.168699, -0.371408, -0.127125, -0.268095, -0.284517, -0.41301, -0.067787, 0.142032, 0.465332, 0.362051, 0.118037, -0.277063, -0.305255, 0.223267, -0.153698, 0.18614, -0.083249, 0.308776, 0.312894, -0.099923, -0.498117, -0.099372, 0.060403, -0.019207, -0.373208, -0.364397, -0.480499, -0.384932, -0.443169, 0.120321, -0.381312, -0.165297, 0.106392, -0.080597, 0.056137, 0.460739, -0.379291, 0.385424, 0.327411, 0.436342, 0.322909, 0.299422, 0.102776, 0.267105, 0.370041, -0.209309, 0.047882, -0.212463, 0.461841, -0.495369, -0.318446, -0.027545, 0.345978, 0.487392, 0.364433, 0.012064, 0.063542, -0.243936, 0.012239, 0.369448, 0.27851, 0.023163, -0.372615, 0.03924, -0.465054, -0.455834, 0.242701, -0.273791, -0.214309, -0.443157, 0.371022, 0.162519, 0.036281, 0.281133, -0.445621, 0.282418, 0.447855, -0.051678, 0.106555, 0.473084, -0.257209, -0.17174, -0.357252, -0.379592, 0.256835, -0.276616, 0.490092, 0.168439, 0.481453, -0.101719, -0.427208, -0.088743, 0.440181, -0.086449, -0.13439, -0.263991, -0.257305, 0.332714, 0.009471, 0.035881, -0.33962, -0.229077, 0.182918, 0.011727, 0.067118, 0.254206, -0.176813, -0.41887, -0.410925, 0.358027, 0.290629, 0.11363, -0.19279, 0.093874, 0.070256, -0.289121, -0.143719, 0.012659, -0.395869, 0.250457, 0.264071, 0.493652, 0.348378, -0.087876, 0.374609, -0.312274, 0.14855, 0.405234, 0.199123, 0.083633, -0.4729, -0.383926, 0.038478, 0.166092, -0.023277, -0.106124, -0.27916, 0.134412, -0.146708, -0.305753, -0.201661, 0.15042, -0.037833, -0.201223, -0.453545, 0.084431, 0.431608, -0.170439, 0.109893, 0.228018, -0.032954, 0.277403, -0.361311, -0.057676, 0.079963, -0.23757, -0.241693, -0.289321, 0.385139, -0.254111, 0.217462, -0.435744, -0.440005, 0.411857, 0.057435, 0.349528, 0.05981, -0.062371, 0.146616, -0.193727, 0.182536, -0.128474, 0.424037, -0.075133, -0.423021, -0.163128, -0.304367, -0.44836, -0.211917, 0.280899, 0.326996, -0.288986, -0.066915, 0.487316, -0.323647, -0.386809, 0.102265, -0.428692, -0.388461, -0.183851, -0.421938, -0.3243, -0.045307, -0.489334, 0.466308, 0.087048, 0.198453, 0.39716, -0.166698, 0.490398, 0.337367, 0.123059, -0.057489, 0.081472, 0.158306, -0.250517, -0.315376, -0.244984, -0.166399, 0.215961, -0.205289, 0.088085, 0.160666, 0.27195, -0.049752, 0.336982, 0.291858, -0.156098, 0.343749, 0.166636, 0.298318, 0.306627, -0.176243, 0.33114, 0.41155, -0.372301, 0.197944, 0.024214, 0.286483, -0.146336, -0.207929, 0.118346, 0.129762, -0.093207, -0.141441, 0.011341, 0.037037, 0.161134, -0.071586, 0.299059, 0.382968, -0.233399, -0.299731, -0.006938, -0.392269, 0.189879, -0.439719, -0.498061, 0.243749, -0.059147, 0.326633, 0.095047, -0.473591, -0.046647, 0.312408, 0.067435, 0.432515, -0.411063, 0.300558, 0.324676, 0.490226, -0.267532, -0.475258, -0.393617, 0.204222, -0.051753, 0.249771, 0.067138, 0.439866, 0.165433, 0.045409, 0.4336, -0.389911, -0.138479, 0.347289, 0.456525, 0.008765, 0.212603, 0.398682, -0.123527, 0.13432, 0.179189, 0.166884, -0.378079, 0.347325, -0.003682, 0.32795, 0.458558, -0.29418, 0.036715, -0.086447, -0.1915, 0.322133, -0.48425, -0.286313, -0.041827, 0.084916, 0.00694, 0.02899, 0.373621, 0.090184, 0.236748, -0.13358, 0.341988, -0.374926, 0.127428, 0.172049, -0.342084, 0.048096, -0.443325, 0.194701, -0.426379, 0.409617, -0.197192, -0.425782, -0.268612, 0.156329, 0.144261, 0.24807, 0.297236, -0.236552, -0.417883, -0.229916, 0.014506, -0.361594, 0.154439, 0.135029, 0.396127, 0.139765, -0.188854, -0.297194, -0.225706, 0.055128, 0.318192, 0.316008, 0.426323, 0.381778, 0.159783, 0.204578, 0.495429, -0.246201, -0.092601, -0.228666, -0.074828, 0.359435, 0.024356, -0.147898, 0.126331, -0.307564, -0.425327, -0.455798] + } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 8, 64], + "data": { + "kind": "values", + "values": [0.372574, -0.306708, 0.184696, 0.272713, -0.148147, -0.122402, 0.033584, -0.125638, 0.227005, -0.30606, 0.136125, 0.167737, 0.018802, -0.24656, -0.205723, 0.118218, 0.497002, 0.044897, -0.050453, -0.078412, -0.326121, 0.042089, 0.441168, 0.458006, -0.149514, 0.059117, 0.301394, -0.209086, 0.059405, 0.322551, 0.181637, -0.070391, 0.247732, -0.27088, -0.496768, -0.425765, -0.455956, -0.242304, 0.426551, 0.061955, 0.453098, -0.495666, 0.232227, 0.233034, 0.322021, 0.309097, 0.208241, 0.435606, -0.078269, -0.05656, -0.363385, 0.387851, 0.487342, -0.173121, -0.026534, 0.116354, 0.061299, 0.241717, -0.183831, -0.310622, 0.483011, 0.01456, -0.248619, -0.036185, -0.1313, -0.366616, -0.322331, 0.064553, -0.074749, 0.268896, -0.303493, -0.184979, -0.058216, -0.331057, 0.165371, -0.162349, -0.197279, 0.394721, 0.006702, -0.456985, -0.133926, 0.367525, -0.133525, -0.273619, 0.050734, 0.194798, 0.216127, -0.009556, -0.02352, 0.244232, -0.008405, -0.287023, -0.499521, -0.065344, -0.136786, -0.221845, 0.494022, 0.212482, -0.269775, -0.209704, 0.203238, -0.317563, -0.466362, -0.011215, 0.037548, 0.361352, -0.018789, -0.417094, 0.129602, -0.476261, 0.278283, -0.274642, -0.489551, -0.392078, 0.458218, -0.156817, 0.243694, -0.34854, 0.406212, -0.455124, 0.362385, 0.224619, 0.1702, -0.177594, -0.047613, 0.330641, -0.347365, -0.233084, 0.225994, 0.241559, 0.276866, -0.483153, -0.070481, -0.26169, -0.278121, -0.342778, -0.07138, -0.276106, 0.319151, -0.211132, 0.315116, -0.321671, 0.205356, 0.486297, -0.120972, -0.075846, -0.175313, 0.363014, 0.289611, 0.17123, -0.125469, 0.462999, -0.170879, 0.309588, 0.288264, -0.302894, 0.023059, -0.080933, 0.348892, -0.456864, 0.01066, 0.464513, -0.472691, -0.271665, -0.000643, -0.354744, 0.138917, 0.113056, -0.353016, -0.132346, 0.412347, 0.494355, -0.284535, 0.309418, 0.064046, -0.451756, -0.16027, -0.455509, -0.027529, -0.370621, -0.244376, 0.197224, 0.045646, -0.353087, -0.169454, -0.405403, -0.124184, -0.264728, 0.240572, -0.114761, 0.274982, -0.061399, -0.362185, -0.406011, 0.285021, -0.372359, -0.348234, 0.080765, -0.243814, -0.132237, 0.068372, -0.127534, 0.087701, 0.456935, 0.030599, 0.336685, 0.224224, -0.147638, 0.104225, -0.334044, -0.498085, -0.295465, -0.447379, -0.358693, 0.437865, 0.217468, -0.335733, 0.143857, 0.056636, 0.053846, 0.476752, -0.116792, -0.152346, 0.003849, -0.258996, 0.311385, -0.052781, 0.136397, 0.46951, 0.191076, -0.361128, -0.348434, -0.087733, -0.213087, -0.234132, 0.152927, 0.30422, 0.148556, 0.149514, -0.441438, 0.227932, 0.454319, -0.392891, 0.369486, -0.06476, -0.115906, -0.336748, -0.485737, 0.471642, -0.009675, -0.451748, 0.488523, -0.052824, 0.035777, -0.422595, 0.008167, -0.202478, 0.266846, -0.495131, -0.105017, 0.379102, -0.40882, -0.054089, 0.41344, -0.2581, -0.37167, 0.461253, -0.2226, -0.076222, -0.397789, 0.397289, 0.045506, -0.389977, 0.044102, 0.075895, 0.12399, -0.498129, -0.22505, 0.427614, 0.095648, 0.237779, 0.135711, -0.338055, -0.406254, 0.211169, -0.065502, 0.068484, 0.439592, 0.128415, 0.07927, -0.457311, 0.26253, -0.349194, 0.364817, -0.070013, 0.32111, -0.035746, 0.3828, -0.21799, 0.244171, -0.379524, 0.023576, -0.381328, -0.089392, 0.493416, -0.461422, 0.289864, 0.186551, 0.194652, -0.368009, -0.469486, -0.261278, -0.208466, -0.449526, 0.302743, -0.357293, 0.171808, -0.360324, 0.018889, 0.00491, -0.101524, 0.336302, 0.239144, 0.31692, -0.101364, 0.064478, -0.062462, -0.23895, -0.169938, 0.005158, -0.069956, 0.185808, 0.469263, 0.207329, -0.441464, -0.029972, -0.035219, 0.009766, 0.22436, 0.494426, 0.299472, 0.242082, -0.134738, 0.439335, -0.194485, 0.173823, -0.030575, -0.372649, 0.075919, -0.319639, 0.220368, -0.37034, 0.116251, -0.405661, 0.366453, -0.145218, 0.080355, -0.288597, 0.408748, 0.338486, 0.433806, 0.418987, -0.298992, -0.046369, -0.111158, 0.416898, 0.055504, 0.258679, -0.04546, 0.498439, 0.448473, -0.29638, 0.445969, -0.139162, 0.418918, 0.412086, -0.20392, -0.086284, -0.245479, 0.25837, -0.485922, 0.054358, -0.441581, 0.458128, -0.365965, 0.378529, 0.290581, 0.391497, 0.497478, -0.204419, -0.122447, -0.122609, -0.286282, -0.483378, 0.484306, -0.382292, -0.358337, 0.384461, 0.041949, 0.030223, 0.236861, 0.014118, 0.125046, 0.18889, -0.392468, 0.212624, 0.272293, 0.090326, 0.123335, -0.353137, 0.344311, -0.034792, 0.420612, 0.228219, 0.01229, -0.146649, 0.147873, 0.182766, -0.028754, 0.161198, -0.204733, 0.275754, -0.028362, 0.42049, 0.451781, 0.159146, 0.263094, -0.033144, -0.166073, 0.310945, -0.456005, -0.242928, 0.429664, -0.464925, 0.010669, 0.043741, 0.101301, 0.490556, -0.400053, 0.126154, -0.008861, 0.218723, 0.018823, -0.103941, 0.073595, 0.137246, -0.345022, 0.397116, 0.197559, 0.469698, -0.038086, 0.116763, -0.037263, -0.420589, 0.412439, 0.229258, 0.273728, 0.176393, -0.385725, -0.491355, -0.084413, -0.133972, -0.318486, 0.130388, -0.005381, 0.008331, 0.455686, 0.080429, -0.411838, 0.356499, -0.329809, 0.390419, 0.447099, 0.443504, -0.348163, -0.082134, 0.094856, 0.185493, -0.346767, -0.110038, 0.166639, -0.329279, 0.409802, -0.070175, 0.33265, -0.397207, 0.015142, -0.21608, -0.35096, 0.366948, -0.291496, 0.469393, -0.319658, 0.188138, -0.331871, 0.060084, 0.237147, 0.349658, 0.404187, -0.17004, 0.072756, -0.058933, -0.154914, -0.027279, 0.382571, 0.104166, -0.289118, -0.383679, 0.449969, 0.245761, 0.267552, 0.351173] + } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 8, 64], + "data": { + "kind": "values", + "values": [0.097056, -0.200735, -0.168325, 0.191973, 0.492785, -0.397293, 0.47762, 0.470182, -0.256314, -0.16696, -0.157802, 0.27402, 0.404261, 0.289624, -0.477124, -0.490144, -0.067094, 0.221887, 0.084429, -0.272333, -0.492298, 0.063575, -0.344991, 0.405092, 0.051202, -0.053699, 0.465574, 0.05767, 0.307941, -0.038854, -0.073572, -0.106016, 0.318435, -0.362953, 0.334614, 0.234486, 0.133419, -0.395259, -0.194075, 0.412745, 0.112058, -0.085, -0.474731, 0.377861, -0.352095, -0.020081, -0.090857, -0.283196, -0.333665, -0.394006, -0.226029, 0.315428, -0.262786, 0.087747, 0.104044, 0.376898, 0.14656, -0.169341, -0.487171, 0.0163, -0.229075, 0.472135, -0.008622, 0.150163, 0.098277, -0.030727, 0.037601, -0.100338, 0.239101, -0.377817, -0.239849, 0.226232, 0.271498, -0.498403, 0.050661, 0.018367, -0.235993, 0.249528, 0.418097, 0.348326, -0.18008, -0.278273, -0.421034, 0.117528, 0.486828, 0.440684, 0.39714, -0.190651, -0.124853, 0.32793, 0.26703, -0.431887, -0.17061, -0.315615, -0.466127, 0.322982, 0.495907, -0.112505, 0.044886, 0.175538, -0.369124, -0.226068, 0.425015, -0.230226, -0.007732, 0.176571, 0.17106, -0.136738, -0.340452, -0.415466, 0.15789, -0.18176, -0.375137, 0.395014, -0.349461, 0.231378, 0.499463, -0.274344, 0.415889, 0.420953, -0.320896, 0.25623, 0.027736, -0.009791, -0.187809, 0.068355, -0.041058, 0.185634, -0.085599, -0.047983, -0.445536, 0.268902, -0.026461, 0.147633, 0.478985, -0.378748, 0.388383, -0.007587, -0.267099, -0.200979, 0.085262, 0.439772, -0.169571, 0.259115, -0.343885, -0.079466, 0.233561, -0.02686, 0.361789, -0.485315, -0.364367, -0.351037, 0.344318, -0.097577, 0.188259, -0.221562, -0.108062, 0.063255, 0.265364, 0.230061, 0.100586, 0.191891, -0.141915, -0.394709, -0.224366, -0.200318, 0.194275, 0.077297, -0.082394, 0.470361, -0.323629, -0.06913, -0.364369, 0.238824, -0.268632, -0.477731, -0.167623, 0.061713, -0.333345, -0.122272, -0.495773, 0.348542, 0.125462, -0.350613, 0.000428, 0.022125, 0.235659, -0.352601, 0.084987, 0.077044, 0.045669, -0.333042, 0.145038, 0.10492, 0.341247, -0.171008, 0.389457, -0.244772, 0.436854, 0.153288, 0.381451, -0.399592, 0.035404, -0.417219, -0.251114, 0.374026, -0.249891, 0.17777, 0.4161, 0.05073, 0.382196, -0.301198, 0.200946, -0.26325, -0.45178, 0.296979, 0.120824, -0.160299, -0.249254, -0.469557, 0.251445, 0.48643, 0.486136, 0.425768, 0.482082, -0.342344, 0.483198, -0.196957, 0.011052, 0.008165, -0.333561, -0.416641, -0.074819, -0.158708, -0.062314, 0.297481, -0.292988, 0.206461, 0.369809, -0.485563, 0.013486, -0.411489, -0.093694, 0.408774, -0.430134, 0.157578, 0.160498, 0.185247, -0.477358, 0.048266, 0.408082, -0.420333, 0.09517, -0.363128, 0.366795, 0.154684, -0.110204, 0.285403, -0.143065, -0.390771, -0.319786, -0.47886, -0.063505, -0.179612, 0.037811, 0.120505, -0.395347, -0.163556, -0.364261, 0.065963, -0.332629, -0.460162, -0.425515, -0.155264, -0.141142, 0.198808, -0.427343, 0.05805, -0.187363, -0.373554, 0.366773, -0.190314, -0.024024, 0.165923, 0.16377, -0.15742, 0.061424, 0.220649, 0.490006, -0.107787, 0.129207, -0.451912, 0.280489, 0.225551, -0.10576, 0.036007, -0.447897, -0.165714, 0.35149, 0.179893, -0.495449, 0.251117, 0.313449, -0.019709, 0.392798, 0.457832, -0.046525, 0.478815, 0.014739, -0.146064, -0.05127, -0.348419, 0.157855, -0.245424, 0.31649, -0.12119, -0.151398, 0.386513, -0.462448, -0.290642, -0.251714, -0.149112, 0.060512, 0.138908, -0.460829, 0.021543, -0.219359, 0.120597, -0.355428, -0.152375, -0.412867, -0.473195, 0.12668, 0.029255, -0.427552, 0.280865, 0.105878, -0.465025, -0.00247, -0.122543, -0.454717, -0.070243, -0.393382, 0.010409, -0.255723, -0.017699, 0.385433, -0.473323, 0.384774, -0.479622, 0.35646, 0.425252, 0.473965, 0.002982, -0.494902, 0.119725, 0.027305, -0.221988, 0.430415, -0.316711, -0.164519, 0.494264, -0.185733, 0.294902, 0.159108, -0.363537, 0.302522, -0.394622, -0.305076, -0.347905, -0.107854, 0.242147, -0.342793, -0.361214, 0.417893, 0.171435, 0.318177, 0.060975, 0.232367, 0.136625, -0.148858, 0.464636, -0.326823, -0.108474, -0.179883, -0.341203, 0.410962, 0.447327, 0.159687, 0.332606, 0.272027, 0.051964, -0.011157, -0.423308, -0.370672, 0.120661, 0.102568, 0.277574, -0.044424, -0.163601, 0.484889, -0.146131, -0.367804, -0.360955, -0.312819, 0.303044, -0.392103, 0.321915, 0.415443, 0.027466, 0.000602, -0.045499, 0.170314, 0.408496, 0.236481, 0.350123, 0.28357, -0.402613, -0.180146, 0.297251, 0.19486, 0.491721, 0.327696, 0.273252, 0.312424, 0.366149, -0.277499, -0.140694, 0.301542, 0.126171, -0.422737, -0.424754, 0.144473, -0.346276, 0.306303, 0.353065, -0.184373, 0.213303, -0.004278, 0.367856, -0.084284, 0.387387, -0.072802, 0.444907, 0.108648, -0.490084, 0.388902, -0.238264, -0.295102, 0.066553, -0.13273, 0.063783, 0.23512, 0.030503, 0.213387, 0.112534, 0.357735, -0.397375, -0.122277, -0.415189, -0.055738, 0.32459, -0.193012, 0.141986, -0.313872, 0.033839, 0.096841, -0.112131, 0.066713, -0.370918, -0.005791, -0.065987, 0.042382, 0.115104, 0.342966, 0.291192, 0.27625, -0.104395, 0.066719, -0.194191, 0.268429, -0.217152, -0.21754, 0.359961, 0.226512, -0.307151, 0.138818, 0.053941, 0.020042, 0.494861, -0.027322, -0.410099, 0.062712, -0.016086, -0.122095, -0.42456, 0.466054, 0.445141, 0.279932, -0.034131, 0.308359, -0.221344, -0.390049, 0.363891, 0.362582, -0.346268, -0.327533, 0.395456, 0.12698, -0.338978, -0.200037, -0.119465] + } + }, + "pastKeyT": { + "dtype": "int8", + "shape": [1, 1, 8, 64], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_quant_int8_large_head_zero_cache" } } + }, + "pastValueT": { + "dtype": "int8", + "shape": [1, 1, 8, 64], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_quant_int8_large_head_zero_cache" } } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [7] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [8] } }, + "kScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [0.0039332] } }, + "vScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [0.0039328] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 8, 128], "tolerance": 0.05 }, + "presentKeyT": { "dtype": "int8", "shape": [1, 1, 8, 64], "tolerance": 0 }, + "presentValueT": { "dtype": "int8", "shape": [1, 1, 8, 64], "tolerance": 0 } + } + }, + { + "name": "ort_quant_int4_gqaratio4_b1q4_h4kv1d16", + "attrs": { + "num_heads": 4, + "kv_num_heads": 1, + "kv_cache_bit_width": 4, + "k_quant_type": "PER_TENSOR", + "v_quant_type": "PER_TENSOR" + }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 4, 64], + "data": { + "kind": "values", + "values": [0.082308, 0.019819, -0.034024, 0.277037, -0.077135, -0.466628, -0.082611, 0.308729, 0.11234, 0.214904, -0.317721, 0.016195, 0.061967, -0.251316, 0.419605, -0.389136, 0.158377, -0.171533, -0.314711, 0.171006, 0.418748, -0.368512, -0.089387, 0.349135, -0.090375, -0.467301, 0.066158, 0.304013, -0.351828, 0.402985, -0.01996, -0.048985, 0.204581, 0.335118, 0.279197, 0.124608, -0.275582, 0.141191, 0.422468, -0.253317, -0.410373, 0.46529, -0.205779, -0.405497, 0.436971, -0.14485, 0.327278, -0.238, -0.247373, -0.363456, -0.37611, 0.408198, -0.238281, -0.065844, 0.043498, -0.113526, -0.168485, -0.317308, -0.273944, -0.151551, 0.107045, -0.028029, 0.01964, 0.416226, -0.402106, -0.43025, 0.281105, 0.01674, 0.434919, -0.033959, -0.481453, -0.028033, -0.484372, -0.217347, -0.155699, 0.438746, 0.256996, -0.108279, 0.20724, -0.208042, -0.325565, 0.086976, 0.373127, 0.044809, -0.494295, -0.348512, -0.378245, 0.399983, -0.40152, -0.334346, 0.129636, -0.474602, 0.280348, 0.399162, -0.493008, -0.35168, -0.070028, -0.369303, 0.457786, 0.061587, 0.468188, 0.239124, 0.031557, 0.135702, -0.012245, -0.090088, 0.435791, 0.430518, 0.240547, 0.300536, -0.122395, 0.000296, 0.200901, -0.491379, 0.40346, 0.485869, 0.423899, 0.18656, 0.205686, 0.087662, 0.417308, 0.146378, 0.364543, 0.013327, -0.254039, -0.201427, 0.494586, 0.49373, 0.08729, 0.268691, 0.264073, -0.038945, -0.043208, -0.46461, 0.256173, 0.482762, 0.322169, 0.433737, 0.140714, 0.045159, -0.175119, -0.132516, -0.374761, -0.219674, -0.172124, 0.074085, 0.154098, -0.269842, 0.008243, 0.286197, -0.175656, 0.241259, -0.286464, -0.36659, 0.487174, -0.415547, -0.199484, -0.372034, -0.496337, -0.107356, 0.17507, 0.078215, 0.123551, 0.010797, -0.24795, -0.226458, -0.044176, 0.35177, 0.053783, -0.215451, 0.49984, 0.351761, 0.166002, -0.277798, 0.205756, -0.380893, 0.269957, -0.200274, 0.032998, 0.370204, 0.469432, 0.040976, -0.024468, 0.246272, 0.131451, 0.479044, -0.477325, 0.104742, 0.459394, -0.311143, -0.475788, -0.121004, -0.48271, 0.497501, 0.428552, 0.100217, 0.424031, 0.21603, 0.362687, 0.175099, -0.063742, 0.236517, 0.022691, 0.013166, 0.138668, 0.338081, -0.156478, 0.097452, 0.460495, -0.454347, 0.416872, -0.407713, -0.263091, -0.114778, -0.374197, -0.418052, -0.46591, -0.120674, 0.464943, 0.033911, 0.456725, 0.126508, 0.014974, -0.419527, -0.343328, -0.191469, -0.459838, -0.287529, 0.261552, 0.469055, 0.17769, 0.478872, -0.057993, -0.092439, 0.187538, 0.285966, 0.336384, -0.467742, 0.231653, -0.110081, -0.44841, 0.015721, -0.348823, 0.176437, 0.458867, -0.402987, 0.210459, -0.178714, 0.00324, -0.12595, 0.276625, 0.425428, 0.24539, 0.429934, -0.09656, 0.218509] + } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 4, 16], + "data": { + "kind": "values", + "values": [0.372574, -0.306708, 0.184696, 0.272713, -0.148147, -0.122402, 0.033584, -0.125638, 0.227005, -0.30606, 0.136125, 0.167737, 0.018802, -0.24656, -0.205723, 0.118218, 0.497002, 0.044897, -0.050453, -0.078412, -0.326121, 0.042089, 0.441168, 0.458006, -0.149514, 0.059117, 0.301394, -0.209086, 0.059405, 0.322551, 0.181637, -0.070391, 0.247732, -0.27088, -0.496768, -0.425765, -0.455956, -0.242304, 0.426551, 0.061955, 0.453098, -0.495666, 0.232227, 0.233034, 0.322021, 0.309097, 0.208241, 0.435606, -0.078269, -0.05656, -0.363385, 0.387851, 0.487342, -0.173121, -0.026534, 0.116354, 0.061299, 0.241717, -0.183831, -0.310622, 0.483011, 0.01456, -0.248619, -0.036185] + } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 4, 16], + "data": { + "kind": "values", + "values": [0.097056, -0.200735, -0.168325, 0.191973, 0.492785, -0.397293, 0.47762, 0.470182, -0.256314, -0.16696, -0.157802, 0.27402, 0.404261, 0.289624, -0.477124, -0.490144, -0.067094, 0.221887, 0.084429, -0.272333, -0.492298, 0.063575, -0.344991, 0.405092, 0.051202, -0.053699, 0.465574, 0.05767, 0.307941, -0.038854, -0.073572, -0.106016, 0.318435, -0.362953, 0.334614, 0.234486, 0.133419, -0.395259, -0.194075, 0.412745, 0.112058, -0.085, -0.474731, 0.377861, -0.352095, -0.020081, -0.090857, -0.283196, -0.333665, -0.394006, -0.226029, 0.315428, -0.262786, 0.087747, 0.104044, 0.376898, 0.14656, -0.169341, -0.487171, 0.0163, -0.229075, 0.472135, -0.008622, 0.150163] + } + }, + "pastKeyT": { + "dtype": "uint8", + "shape": [1, 1, 4, 8], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_quant_int8_pertensor_b1q4_h2kv1d8_input_pastKeyT" } + } + }, + "pastValueT": { + "dtype": "uint8", + "shape": [1, 1, 4, 8], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_quant_int8_pertensor_b1q4_h2kv1d8_input_pastKeyT" } + } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [3] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } }, + "kScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [0.0710003] } }, + "vScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [0.0703979] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 4, 64], "tolerance": 0.15 }, + "presentKeyT": { "dtype": "uint8", "shape": [1, 1, 4, 8], "tolerance": 0 }, + "presentValueT": { "dtype": "uint8", "shape": [1, 1, 4, 8], "tolerance": 0 } + } + }, + { + "name": "ort_softcap_decode_b1q1p8_h2kv1d8", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "softcap": 20 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 16], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_sharedkv_decode_b1q1p8_h2kv1d8_input_queryT" } + } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 8], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 8], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_key_t_pattern" } } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_value_t_pattern" } } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [7] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [8] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 16], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 } + } + }, + { + "name": "ort_smooth_decode_b1q1p8_h2kv1d8", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "smooth_softmax": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 16], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_sharedkv_decode_b1q1p8_h2kv1d8_input_queryT" } + } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 8], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 8], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_key_t_pattern" } } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_value_t_pattern" } } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [7] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [8] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 16], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 } + } + }, + { + "name": "ort_softcap_smooth_prompt_b1q8p8_h2kv1d8", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "softcap": 30, "smooth_softmax": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 8, 16], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_query_t_pattern" } } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 8], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 8], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_key_t_pattern" } } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_value_t_pattern" } } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [7] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [8] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 8, 16], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 } + } + }, + { + "name": "ort_smooth_gqa8_b1q1p4_h8kv1d16", + "attrs": { "num_heads": 8, "kv_num_heads": 1, "smooth_softmax": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_query_t_pattern" } } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 16], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 16], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 4, 16], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_key_t_pattern" } } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 4, 16], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_value_t_pattern" } } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [3] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 128], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 4, 16], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 4, 16], "tolerance": 0.0001 } + } + }, + { + "name": "ort_softcap_quant_int8_b1q4_h2kv1d8", + "attrs": { + "num_heads": 2, + "kv_num_heads": 1, + "kv_cache_bit_width": 8, + "softcap": 15, + "k_quant_type": "PER_TENSOR", + "v_quant_type": "PER_TENSOR" + }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 4, 16], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_quant_query_t" } } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 4, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_quant_key_t" } } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 4, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_quant_value_t" } } + }, + "pastKeyT": { + "dtype": "int8", + "shape": [1, 1, 4, 8], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_quant_int8_pertensor_b1q4_h2kv1d8_input_pastKeyT" } + } + }, + "pastValueT": { + "dtype": "int8", + "shape": [1, 1, 4, 8], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_quant_int8_pertensor_b1q4_h2kv1d8_input_pastKeyT" } + } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [3] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } }, + "kScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [0.0039134] } }, + "vScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [0.0038802] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 4, 16], "tolerance": 0.05 }, + "presentKeyT": { "dtype": "int8", "shape": [1, 1, 4, 8], "tolerance": 0 }, + "presentValueT": { "dtype": "int8", "shape": [1, 1, 4, 8], "tolerance": 0 } + } + }, + { + "name": "ort_bias_decode_b1q1p8_h2kv1d8", + "attrs": { "num_heads": 2, "kv_num_heads": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 16], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_sharedkv_decode_b1q1p8_h2kv1d8_input_queryT" } + } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 8], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 8], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_key_t_pattern" } } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_value_t_pattern" } } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [7] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [8] } }, + "attentionBiasT": { + "dtype": "float32", + "shape": [1, 2, 1, 8], + "data": { + "kind": "values", + "values": [-0.05, -0.0020574, 0.0341471, 0.0497495, 0.0409297, 0.0098472, -0.035888, -0.0850783, -0.1256802, -0.147753, -0.1458924, -0.120554, -0.0779415, -0.028488, 0.0156987, 0.0438] + } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 16], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 } + } + }, + { + "name": "ort_bias_bcast_prompt_b1q8p8_h2kv1d8", + "attrs": { "num_heads": 2, "kv_num_heads": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 8, 16], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_query_t_pattern" } } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 8], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 8], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_key_t_pattern" } } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_value_t_pattern" } } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [7] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [8] } }, + "attentionBiasT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { + "kind": "values", + "values": [-0.05, -0.0020574, 0.0341471, 0.0497495, 0.0409297, 0.0098472, -0.035888, -0.0850783, -0.1256802, -0.147753, -0.1458924, -0.120554, -0.0779415, -0.028488, 0.0156987, 0.0438, 0.0489358, 0.0298487, -0.0087882, -0.0575151, -0.1044021, -0.1379696, -0.149999, -0.1375452, -0.1036573, -0.0566322, -0.0079833, 0.0303784, 0.0490607, 0.0434895, 0.0150288, -0.0293533, -0.0787903, -0.1211785, -0.1461397, -0.1475626, -0.1250987, -0.0842481, -0.0350123, 0.010554, 0.0412945, 0.049683, 0.0336656, -0.0028361, -0.0508851, -0.0987175, -0.134622, -0.1498082, -0.1405578, -0.1091358, -0.0632352, -0.0140942, 0.0262558, 0.0479358, 0.0456376, 0.019924, -0.0229094, -0.0723756, -0.1163634, -0.1441031, -0.1488032, -0.1293127, -0.0904038, -0.0416026] + } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 8, 16], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 } + } + }, + { + "name": "ort_headsink_decode_b1q1p8_h2kv1d8", + "attrs": { "num_heads": 2, "kv_num_heads": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 16], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_sharedkv_decode_b1q1p8_h2kv1d8_input_queryT" } + } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 8], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 8], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_key_t_pattern" } } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_value_t_pattern" } } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [7] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [8] } }, + "headSinkT": { "dtype": "float32", "shape": [2], "data": { "kind": "values", "values": [0.2, 0.3] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 16], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 } + } + }, + { + "name": "ort_bias_headsink_prompt_b1q4p4_h2kv1d8", + "attrs": { "num_heads": 2, "kv_num_heads": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 4, 16], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_sharedkv_rotary_multibatch_b2q1p8_h2kv1d16_input_queryT" } + } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 8], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 8], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 4, 8], + "data": { + "kind": "values", + "values": [0.2, 0.4, 0.6, 0.8, 1.0, 0.2, 0.4, 0.6, 0.8, 1.0, 0.2, 0.4, 0.6, 0.8, 1.0, 0.2, 0.4, 0.6, 0.8, 1.0, 0.2, 0.4, 0.6, 0.8, 1.0, 0.2, 0.4, 0.6, 0.8, 1.0, 0.2, 0.4] + } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 4, 8], + "data": { + "kind": "values", + "values": [0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6, 0.9, 0.3, 0.6] + } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [3] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } }, + "attentionBiasT": { + "dtype": "float32", + "shape": [1, 2, 4, 4], + "data": { + "kind": "values", + "values": [-0.05, -0.0020574, 0.0341471, 0.0497495, 0.0409297, 0.0098472, -0.035888, -0.0850783, -0.1256802, -0.147753, -0.1458924, -0.120554, -0.0779415, -0.028488, 0.0156987, 0.0438, 0.0489358, 0.0298487, -0.0087882, -0.0575151, -0.1044021, -0.1379696, -0.149999, -0.1375452, -0.1036573, -0.0566322, -0.0079833, 0.0303784, 0.0490607, 0.0434895, 0.0150288, -0.0293533] + } + }, + "headSinkT": { "dtype": "float32", "shape": [2], "data": { "kind": "values", "values": [0.2, 0.3] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 4, 16], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 4, 8], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 4, 8], "tolerance": 0.0001 } + } + }, + { + "name": "flashprefill_pastzero_h8kv2_d64_q32", + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 32, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 32, 512], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 32, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 32, 64], "tolerance": 0.0001 } + } + }, + { + "name": "flashprefill_pastzero_h8kv2_d128_q16", + "provenance": { + "notes": "Compact companion for bench case sharedkv-prefill-32h8kv-d128-s64, preserving shared-KV prefill with headDim=128." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.08838834764831845 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 16, 1024], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 256], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 256], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 16, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 16, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [15] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [16] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 16, 1024], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 16, 128], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 16, 128], "tolerance": 0.0001 } + } + }, + { + "name": "flashprefill_pastlen_h8kv2_d64_q16p48", + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 16, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 48, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 48, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [47] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [48] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 16, 512], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 48, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 48, 64], "tolerance": 0.0001 } + } + }, + { + "name": "flashprefill_window_h8kv2_d64_q32_w16", + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125, "local_window_size": 16 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 32, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 32, 512], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 32, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 32, 64], "tolerance": 0.0001 } + } + }, + { + "name": "flashprefill_rotary_h8kv2_d64_q32", + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125, "do_rotary": 1, "local_window_size": 16 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 32, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } }, + "cosCacheT": { + "dtype": "float32", + "shape": [32, 32], + "data": { "kind": "fillFloat32", "scale": 0.5, "sinStep": 0.031, "cosStep": 0.007 } + }, + "sinCacheT": { + "dtype": "float32", + "shape": [32, 32], + "data": { "kind": "fillFloat32", "scale": 0.5, "sinStep": 0.013, "cosStep": 0.037 } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 32, 512], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 32, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 32, 64], "tolerance": 0.0001 } + } + }, + { + "name": "qnorm_rotary_scalar_h8kv2_d256_q1", + "provenance": { + "source": "scope coverage", + "test": "rendered-Jinja scope coverage", + "notes": "headDim=256 requires 34,048 bytes for the cooperative path, exceeding a 32 KiB workgroup-storage tier and exercising the thread-per-query Q-norm route." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.0625, "do_rotary": 1, "qk_norm_epsilon": 0.00001 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 2048], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 512], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 512], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 32, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 32, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } }, + "cosCacheT": { + "dtype": "float32", + "shape": [32, 128], + "data": { "kind": "fillFloat32", "scale": 0.5, "sinStep": 0.031, "cosStep": 0.007 } + }, + "sinCacheT": { + "dtype": "float32", + "shape": [32, 128], + "data": { "kind": "fillFloat32", "scale": 0.5, "sinStep": 0.013, "cosStep": 0.037 } + }, + "qNormWeightT": { + "dtype": "float32", + "shape": [256], + "data": { "kind": "fillFloat32", "scale": 0.3, "sinStep": 0.05, "cosStep": 0.09 } + }, + "kNormWeightT": { + "dtype": "float32", + "shape": [256], + "data": { "kind": "fillFloat32", "scale": 0.3, "sinStep": 0.05, "cosStep": 0.09 } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 2048], "tolerance": 0.0003 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 32, 256], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 32, 256], "tolerance": 0.0001 } + } + }, + { + "name": "f16_qnorm_rotary_scalar_f16wts_h8kv2_d256_q1", + "provenance": { + "source": "scope coverage", + "test": "rendered-Jinja scope coverage", + "notes": "Float16 counterpart to the thread-per-query Q-norm case. It verifies explicit f32 conversion of q_norm_weight, cos_cache, and sin_cache before scalar arithmetic." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.0625, "do_rotary": 1, "qk_norm_epsilon": 0.00001 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 1, 2048], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { "dtype": "float16", "shape": [1, 0, 512], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float16", "shape": [1, 0, 512], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float16", + "shape": [1, 2, 32, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float16", + "shape": [1, 2, 32, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } }, + "cosCacheT": { + "dtype": "float16", + "shape": [32, 128], + "data": { "kind": "fillFloat32", "scale": 0.5, "sinStep": 0.031, "cosStep": 0.007 } + }, + "sinCacheT": { + "dtype": "float16", + "shape": [32, 128], + "data": { "kind": "fillFloat32", "scale": 0.5, "sinStep": 0.013, "cosStep": 0.037 } + }, + "qNormWeightT": { + "dtype": "float16", + "shape": [256], + "data": { "kind": "fillFloat32", "scale": 0.3, "sinStep": 0.05, "cosStep": 0.09 } + }, + "kNormWeightT": { + "dtype": "float16", + "shape": [256], + "data": { "kind": "fillFloat32", "scale": 0.3, "sinStep": 0.05, "cosStep": 0.09 } + } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 1, 2048], "tolerance": 0.0003 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 2, 32, 256], "tolerance": 0.001 }, + "presentValueT": { "dtype": "float16", "shape": [1, 2, 32, 256], "tolerance": 0.001 } + } + }, + { + "name": "flashprefill_qnorm_rotary_h8kv2_d64_q32", + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125, "do_rotary": 1, "local_window_size": 16 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 32, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } }, + "cosCacheT": { + "dtype": "float32", + "shape": [32, 32], + "data": { "kind": "fillFloat32", "scale": 0.5, "sinStep": 0.031, "cosStep": 0.007 } + }, + "sinCacheT": { + "dtype": "float32", + "shape": [32, 32], + "data": { "kind": "fillFloat32", "scale": 0.5, "sinStep": 0.013, "cosStep": 0.037 } + }, + "qNormWeightT": { + "dtype": "float32", + "shape": [64], + "data": { "kind": "fillFloat32", "scale": 0.3, "sinStep": 0.05, "cosStep": 0.09 } + }, + "kNormWeightT": { + "dtype": "float32", + "shape": [64], + "data": { "kind": "fillFloat32", "scale": 0.3, "sinStep": 0.05, "cosStep": 0.09 } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 32, 512], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 32, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 32, 64], "tolerance": 0.0001 } + } + }, + { + "name": "flashdecode_splitk_h8kv2_d64_kv1024", + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 1024, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 1024, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [1023] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [1024] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 512], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 1024, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 1024, 64], "tolerance": 0.0001 } + } + }, + { + "name": "flashdecode_splitk_h8kv2_d128_kv1024", + "provenance": { + "notes": "Compact companion for bench case sharedkv-decode-32h8kv-d128-kv2048, preserving f32 shared-past decode with headDim=128 and the split-K route." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.08838834764831845 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 1024], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 256], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 256], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 1024, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 1024, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [1023] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [1024] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 1024], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 1024, 128], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 1024, 128], "tolerance": 0.0001 } + } + }, + { + "name": "f16_decode_cooperative_h8kv2_d128_kv512", + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.08838834764831845 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 1, 1024], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { "dtype": "float16", "shape": [1, 0, 256], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float16", "shape": [1, 0, 256], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float16", + "shape": [1, 2, 512, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float16", + "shape": [1, 2, 512, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [511] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [512] } } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 1, 1024], "tolerance": 0.03 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 2, 512, 128], "tolerance": 0.001 }, + "presentValueT": { "dtype": "float16", "shape": [1, 2, 512, 128], "tolerance": 0.001 } + } + }, + { + "name": "f16_decode_splitk_h8kv2_d128_kv1024", + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.08838834764831845 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 1, 1024], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { "dtype": "float16", "shape": [1, 0, 256], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float16", "shape": [1, 0, 256], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float16", + "shape": [1, 2, 1024, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float16", + "shape": [1, 2, 1024, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [1023] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [1024] } } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 1, 1024], "tolerance": 0.03 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 2, 1024, 128], "tolerance": 0.001 }, + "presentValueT": { "dtype": "float16", "shape": [1, 2, 1024, 128], "tolerance": 0.001 } + } + }, + { + "name": "f16_prefill_flash_h8kv2_d128_s256", + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.08838834764831845 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 256, 1024], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { "dtype": "float16", "shape": [1, 0, 256], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float16", "shape": [1, 0, 256], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float16", + "shape": [1, 2, 256, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float16", + "shape": [1, 2, 256, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [255] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [256] } } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 256, 1024], "tolerance": 0.03 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 2, 256, 128], "tolerance": 0.001 }, + "presentValueT": { "dtype": "float16", "shape": [1, 2, 256, 128], "tolerance": 0.001 } + } + }, + { + "name": "f16_rotary_decode_f16cos_h8kv2_d64_kv512", + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125, "do_rotary": 1 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 1, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { "dtype": "float16", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float16", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float16", + "shape": [1, 2, 512, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float16", + "shape": [1, 2, 512, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [511] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [512] } }, + "cosCacheT": { + "dtype": "float16", + "shape": [512, 32], + "data": { "kind": "fillFloat32", "scale": 0.5, "sinStep": 0.031, "cosStep": 0.007 } + }, + "sinCacheT": { + "dtype": "float16", + "shape": [512, 32], + "data": { "kind": "fillFloat32", "scale": 0.5, "sinStep": 0.013, "cosStep": 0.037 } + } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 1, 512], "tolerance": 0.03 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 2, 512, 64], "tolerance": 0.001 }, + "presentValueT": { "dtype": "float16", "shape": [1, 2, 512, 64], "tolerance": 0.001 } + } + }, + { + "name": "f16_rotary_prefill_f16cos_h8kv2_d64_q32", + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125, "do_rotary": 1 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 32, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { "dtype": "float16", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float16", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float16", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float16", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } }, + "cosCacheT": { + "dtype": "float16", + "shape": [32, 32], + "data": { "kind": "fillFloat32", "scale": 0.5, "sinStep": 0.031, "cosStep": 0.007 } + }, + "sinCacheT": { + "dtype": "float16", + "shape": [32, 32], + "data": { "kind": "fillFloat32", "scale": 0.5, "sinStep": 0.013, "cosStep": 0.037 } + } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 32, 512], "tolerance": 0.03 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 2, 32, 64], "tolerance": 0.001 }, + "presentValueT": { "dtype": "float16", "shape": [1, 2, 32, 64], "tolerance": 0.001 } + } + }, + { + "name": "f16_qnorm_rotary_prefill_f16wts_h8kv2_d64_q32", + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125, "do_rotary": 1, "qk_norm_epsilon": 0.00001 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 32, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { "dtype": "float16", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float16", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float16", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float16", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } }, + "cosCacheT": { + "dtype": "float16", + "shape": [32, 32], + "data": { "kind": "fillFloat32", "scale": 0.5, "sinStep": 0.031, "cosStep": 0.007 } + }, + "sinCacheT": { + "dtype": "float16", + "shape": [32, 32], + "data": { "kind": "fillFloat32", "scale": 0.5, "sinStep": 0.013, "cosStep": 0.037 } + }, + "qNormWeightT": { + "dtype": "float16", + "shape": [64], + "data": { "kind": "fillFloat32", "scale": 0.3, "sinStep": 0.05, "cosStep": 0.09 } + }, + "kNormWeightT": { + "dtype": "float16", + "shape": [64], + "data": { "kind": "fillFloat32", "scale": 0.3, "sinStep": 0.05, "cosStep": 0.09 } + } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 32, 512], "tolerance": 0.03 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 2, 32, 64], "tolerance": 0.001 }, + "presentValueT": { "dtype": "float16", "shape": [1, 2, 32, 64], "tolerance": 0.001 } + } + }, + { + "name": "newkv_past_plain_h4kv2_d16_q4p8", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "scale": 0.25 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 4, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 4, 32], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.013, "cosStep": 0.017 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 4, 32], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.019, "cosStep": 0.023 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 8, 16], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 8, 16], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [11] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [12] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 4, 64], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 12, 16], "tolerance": 0.001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 12, 16], "tolerance": 0.001 } + } + }, + { + "name": "newkv_past_flash_h8kv2_d64_q32p32", + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 32, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 32, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.013, "cosStep": 0.017 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 32, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.019, "cosStep": 0.023 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [63] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [64] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 32, 512], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 64, 64], "tolerance": 0.001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 64, 64], "tolerance": 0.001 } + } + }, + { + "name": "newkv_past_splitk_h8kv2_d64_q1p1024", + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.013, "cosStep": 0.017 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.019, "cosStep": 0.023 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 1024, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 1024, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [1024] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [1025] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 512], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 1025, 64], "tolerance": 0.001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 1025, 64], "tolerance": 0.001 } + } + }, + { + "name": "newkv_past_f16_flash_h8kv2_d64_q32p32", + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 32, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float16", + "shape": [1, 32, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.013, "cosStep": 0.017 } + }, + "valueT": { + "dtype": "float16", + "shape": [1, 32, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.019, "cosStep": 0.023 } + }, + "pastKeyT": { + "dtype": "float16", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float16", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [63] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [64] } } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 32, 512], "tolerance": 0.03 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 2, 64, 64], "tolerance": 0.001 }, + "presentValueT": { "dtype": "float16", "shape": [1, 2, 64, 64], "tolerance": 0.001 } + } + }, + { + "name": "scalar_fallback_defaultscale_gqa4_hd32_q4_s4", + "provenance": { + "notes": "Portable scalar direct-QKV route using the ONNX Runtime provider's default-scale sentinel." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 2, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 4, 256], + "data": { "kind": "fillFloat32", "scale": 0.4, "sinStep": 0.013, "cosStep": 0.021 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 4, 64], + "data": { "kind": "fillFloat32", "scale": 0.4, "sinStep": 0.017, "cosStep": 0.019 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 4, 64], + "data": { "kind": "fillFloat32", "scale": 0.4, "sinStep": 0.023, "cosStep": 0.011 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [3] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 4, 256] }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 4, 32] }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 4, 32] } + } + }, + { + "name": "scalar_fallback_window1_decode_hd32_h4kv2_kv16", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "scale": 0.17677669529663687, "local_window_size": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "scale": 0.4, "sinStep": 0.013, "cosStep": 0.021 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 16, 64], + "data": { "kind": "fillFloat32", "scale": 0.4, "sinStep": 0.017, "cosStep": 0.019 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 16, 64], + "data": { "kind": "fillFloat32", "scale": 0.4, "sinStep": 0.023, "cosStep": 0.011 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [15] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [16] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 128] }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 16, 32] }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 16, 32] } + } + }, + { + "name": "scalar_fallback_unaligned_hd66_h4kv2_s8", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "scale": 0.12309149097933272, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 8, 264], + "data": { "kind": "fillFloat32", "scale": 0.3, "sinStep": 0.013, "cosStep": 0.021 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 8, 132], + "data": { "kind": "fillFloat32", "scale": 0.3, "sinStep": 0.017, "cosStep": 0.019 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 8, 132], + "data": { "kind": "fillFloat32", "scale": 0.3, "sinStep": 0.023, "cosStep": 0.011 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [7] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [8] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 8, 264] }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 8, 66] }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 8, 66] } + } + }, + { + "name": "flashprefill_bias_h8kv2_d64_q32", + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125, "local_window_size": 16 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 32, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } }, + "attentionBiasT": { + "dtype": "float32", + "shape": [1, 8, 32, 32], + "data": { "kind": "fillFloat32", "scale": 0.1, "sinStep": 0.015, "cosStep": 0.021 } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 32, 512], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 32, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 32, 64], "tolerance": 0.0001 } + }, + "provenance": { + "notes": "Covers additive attention bias on past-KV prefill at headDim=64 and qSeq=32. The bias composes with GQA causal masking through the tiled score path." + } + }, + { + "name": "q31_kv511_h8_kv2_d64_double_threshold_compact", + "provenance": { + "notes": "Compact GQA lock immediately below both flash admission boundaries: qSeq*heads=31*8=248 (<256) and kvSeq=511 (<512). Preserves an 8:2 grouped-query layout and present-K/V outputs." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 31, 512], + "data": { "kind": "fillFloat32", "scale": 0.08, "sinStep": 0.017, "cosStep": 0.031 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 511, 128], + "data": { "kind": "fillFloat32", "scale": 0.08, "sinStep": 0.011, "cosStep": 0.023 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 511, 128], + "data": { "kind": "fillFloat32", "scale": 0.08, "sinStep": 0.007, "cosStep": 0.041 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [510] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [511] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 31, 512], "tolerance": 0.0005, "relTolerance": 0.0005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 511, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 511, 64], "tolerance": 0.000001 } + } + }, + { + "name": "flash_cluster_q31_h8_kv8_d32_compact", + "provenance": { "notes": "Correctness lock for the vectorized headDim=32 GQA prefill cluster path." }, + "attrs": { "num_heads": 8, "kv_num_heads": 8, "scale": 0.17677669529663687, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 31, 256], + "data": { "kind": "fillFloat32", "scale": 0.08, "sinStep": 0.017, "cosStep": 0.031 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 64, 256], + "data": { "kind": "fillFloat32", "scale": 0.08, "sinStep": 0.011, "cosStep": 0.023 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 64, 256], + "data": { "kind": "fillFloat32", "scale": 0.08, "sinStep": 0.007, "cosStep": 0.041 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [63] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [64] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 31, 256], "tolerance": 0.0005, "relTolerance": 0.0005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 8, 64, 32], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 8, 64, 32], "tolerance": 0.000001 } + } + }, + { + "name": "flash_splitk_decode_h8_kv8_d32_kv512_compact", + "provenance": { + "notes": "Correctness lock for headDim=32 GQA decode split-K on subgroup and subgroup-free tiers." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 8, "scale": 0.17677669529663687, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 256], + "data": { "kind": "fillFloat32", "scale": 0.08, "sinStep": 0.013, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 512, 256], + "data": { "kind": "fillFloat32", "scale": 0.08, "sinStep": 0.019, "cosStep": 0.037 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 512, 256], + "data": { "kind": "fillFloat32", "scale": 0.08, "sinStep": 0.029, "cosStep": 0.017 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [511] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [512] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 256], "tolerance": 0.0005, "relTolerance": 0.0005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 8, 512, 32], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 8, 512, 32], "tolerance": 0.000001 } + } + }, + { + "name": "quant_int8_splitk_decode_h2_kv1_d64_kv1024", + "provenance": { + "notes": "Correctness lock for buffer-sharing long-context INT8 decode split-K on subgroup and subgroup-free tiers, including per-channel cache scales, a partially filled physical cache, and an appended K/V token." + }, + "attrs": { + "num_heads": 2, + "kv_num_heads": 1, + "kv_cache_bit_width": 8, + "scale": 0.125, + "k_quant_type": "PER_CHANNEL", + "v_quant_type": "PER_CHANNEL", + "local_window_size": 256 + }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "scale": 0.12, "sinStep": 0.013, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 64], + "data": { "kind": "fillFloat32", "scale": 0.12, "sinStep": 0.017, "cosStep": 0.031 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 64], + "data": { "kind": "fillFloat32", "scale": 0.12, "sinStep": 0.019, "cosStep": 0.037 } + }, + "pastKeyT": { + "dtype": "int8", + "shape": [1, 1, 1024, 64], + "data": { "kind": "cycle", "values": [-8, -3, 0, 2, 7, 4, -6] } + }, + "pastValueT": { + "dtype": "int8", + "shape": [1, 1, 1024, 64], + "data": { "kind": "cycle", "values": [5, -7, 1, 0, 6, -2, 3] } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [767] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [768] } }, + "kScaleT": { + "dtype": "float32", + "shape": [64], + "data": { "kind": "cycle", "values": [0.04, 0.05, 0.0625, 0.075, 0.08] } + }, + "vScaleT": { + "dtype": "float32", + "shape": [64], + "data": { "kind": "cycle", "values": [0.045, 0.055, 0.065, 0.07, 0.085] } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 128], "tolerance": 0.005, "relTolerance": 0.005 }, + "presentKeyT": { "dtype": "int8", "shape": [1, 1, 1024, 64], "tolerance": 0 }, + "presentValueT": { "dtype": "int8", "shape": [1, 1, 1024, 64], "tolerance": 0 } + } + }, + { + "name": "quant_int8_splitk_decode_h2_kv1_d80_nondivisible_value_lanes", + "provenance": { + "notes": "INT8 split-K decode with headDim/4=20, which does not divide the 256-lane workgroup. This pins the ordinary value-accumulation rail when j-split accumulation is unavailable." + }, + "attrs": { + "num_heads": 2, + "kv_num_heads": 1, + "kv_cache_bit_width": 8, + "scale": 0.11180339887498948, + "k_quant_type": "PER_CHANNEL", + "v_quant_type": "PER_CHANNEL" + }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 160], + "data": { "kind": "fillFloat32", "scale": 0.12, "sinStep": 0.013, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 80], + "data": { "kind": "fillFloat32", "scale": 0.12, "sinStep": 0.017, "cosStep": 0.031 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 80], + "data": { "kind": "fillFloat32", "scale": 0.12, "sinStep": 0.019, "cosStep": 0.037 } + }, + "pastKeyT": { + "dtype": "int8", + "shape": [1, 1, 1024, 80], + "data": { "kind": "cycle", "values": [-8, -3, 0, 2, 7, 4, -6] } + }, + "pastValueT": { + "dtype": "int8", + "shape": [1, 1, 1024, 80], + "data": { "kind": "cycle", "values": [5, -7, 1, 0, 6, -2, 3] } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [767] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [768] } }, + "kScaleT": { + "dtype": "float32", + "shape": [80], + "data": { "kind": "cycle", "values": [0.04, 0.05, 0.0625, 0.075, 0.08] } + }, + "vScaleT": { + "dtype": "float32", + "shape": [80], + "data": { "kind": "cycle", "values": [0.045, 0.055, 0.065, 0.07, 0.085] } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 160], "tolerance": 0.005, "relTolerance": 0.005 }, + "presentKeyT": { "dtype": "int8", "shape": [1, 1, 1024, 80], "tolerance": 0 }, + "presentValueT": { "dtype": "int8", "shape": [1, 1, 1024, 80], "tolerance": 0 } + } + }, + { + "name": "qkv_prefill_f32_h8kv2_d256_register_boundary", + "provenance": { + "notes": "Upper-head routing boundary for QKV prefill. On subgroup tiers the generic online kernel beats both register-heavy Q32 broadcast and the clustered d256 topology; no-subgroup tiers retain the shared-memory cluster as their fastest fallback." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.0625, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 32, 2048], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 32, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 32, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 32, 2048], "tolerance": 0.00015, "relTolerance": 0.00015 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 32, 256], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 32, 256], "tolerance": 0.000001 } + } + }, + { + "name": "ort_sharedkv_decode_capacity_slack_b1q1p8_h2kv1d8", + "attrs": { "num_heads": 2, "kv_num_heads": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 16], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_sharedkv_decode_b1q1p8_h2kv1d8_input_queryT" } + } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 8], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 8], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_key_t_pattern" } } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_value_t_pattern" } } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [3] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [8] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 16], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/webgpu/bert/flash_attention.wgsl.template", + "test": "get_total_sequence_length / per-batch seqlens_k (PRs #29247, #29002)", + "notes": "past_present_share_buffer semantics: the BNSH cache dim2 is capacity, seqlens_k[b]+1 is the active end per batch. Rows past the active end are storage and must not be attended; rotary positions derive from the active end, not the capacity." + } + }, + { + "name": "ort_sharedkv_decode_right_padded_b2q1p4_h2kv1d8", + "attrs": { "num_heads": 2, "kv_num_heads": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [2, 1, 16], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_sharedkv_batched_b2q1p4_h2kv1d8_input_queryT" } + } + }, + "keyT": { "dtype": "float32", "shape": [2, 0, 8], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [2, 0, 8], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [2, 1, 4, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_key_t_pattern" } } + }, + "pastValueT": { + "dtype": "float32", + "shape": [2, 1, 4, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_past_value_t_pattern" } } + }, + "seqlensKT": { "dtype": "int32", "shape": [2], "data": { "kind": "values", "values": [3, 1] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [2, 1, 16], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [2, 1, 4, 8], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [2, 1, 4, 8], "tolerance": 0.0001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/webgpu/bert/flash_attention.wgsl.template", + "test": "get_total_sequence_length / per-batch seqlens_k (PRs #29247, #29002)", + "notes": "past_present_share_buffer semantics: the BNSH cache dim2 is capacity, seqlens_k[b]+1 is the active end per batch. Rows past the active end are storage and must not be attended; rotary positions derive from the active end, not the capacity." + } + }, + { + "name": "ort_sharedkv_rotary_decode_capacity_slack_b1q1p8_h2kv1d16", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "do_rotary": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 32], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_sharedkv_batched_b2q1p4_h2kv1d8_input_queryT" } + } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 16], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 16], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 8, 16], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_sharedkv_rotary_decode_b1q1p8_h2kv1d16_input_pastKeyT" } + } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 8, 16], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_sharedkv_rotary_decode_b1q1p8_h2kv1d16_input_pastValueT" } + } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [8] } }, + "cosCacheT": { + "dtype": "float32", + "shape": [24, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_rotary_cos_cache_t" } } + }, + "sinCacheT": { + "dtype": "float32", + "shape": [24, 8], + "data": { "kind": "values", "values": { "$ref": "#/fixtureArrays/ort_rotary_sin_cache_t" } } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 32], "tolerance": 0.001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 8, 16], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 8, 16], "tolerance": 0.0001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/webgpu/bert/flash_attention.wgsl.template", + "test": "get_total_sequence_length / per-batch seqlens_k (PRs #29247, #29002)", + "notes": "past_present_share_buffer semantics: the BNSH cache dim2 is capacity, seqlens_k[b]+1 is the active end per batch. Rows past the active end are storage and must not be attended; rotary positions derive from the active end, not the capacity." + } + }, + { + "name": "sharedkv_decode_splitk_capacity_slack_b1q1p1024_h2kv1d64", + "attrs": { "num_heads": 2, "kv_num_heads": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.11, "cosStep": 0.29, "scale": 0.5 } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 64], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 64], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 1024, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.31, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 1024, 64], + "data": { "kind": "linspace", "start": 0.0, "end": 4.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [255] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 128], "tolerance": 0.001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 1024, 64], "tolerance": 0.001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 1024, 64], "tolerance": 0.001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/webgpu/bert/flash_attention_decode_qkv.wgsl.template", + "test": "seqlens_k-driven total_sequence_length in the decode split path", + "notes": "Cache capacity 1024 with active end 512 selects the float decode split-K variant; the split ranges must cover only the active prefix." + } + }, + { + "name": "sharedkv_decode_tiny_headdim8_kv512", + "provenance": { + "source": "onnxruntime/contrib_ops/webgpu/bert/flash_attention_decode_qkv.wgsl.template", + "test": "head_size=8 split-reduce OOB race class (PR #29593)", + "notes": "head_size below the flash minimum routes to the cooperative fallback; this pins the tiny-headDim decode regime where tile-width-guarded reductions have raced in other engines." + }, + "attrs": { "num_heads": 4, "kv_num_heads": 2 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 32], + "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.29, "scale": 0.5 } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 16], "data": { "kind": "constant", "value": 0.0 } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 16], "data": { "kind": "constant", "value": 0.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 512, 8], + "data": { "kind": "fillFloat32", "sinStep": 0.11, "cosStep": 0.23, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 512, 8], + "data": { "kind": "fillFloat32", "sinStep": 0.07, "cosStep": 0.31, "scale": 0.5 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [511] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [512] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 32], "tolerance": 0.0005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 512, 8], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 512, 8], "tolerance": 0.0001 } + } + }, + { + "name": "sharedkv_decode_tiny_headdim16_kv512", + "provenance": { + "source": "onnxruntime/contrib_ops/webgpu/bert/flash_attention_decode_qkv.wgsl.template", + "test": "head_size=16 split-reduce OOB race class (PR #29593)", + "notes": "Sibling of the head_size=8 case at the next tiny head width." + }, + "attrs": { "num_heads": 4, "kv_num_heads": 2 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.29, "scale": 0.5 } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 32], "data": { "kind": "constant", "value": 0.0 } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 32], "data": { "kind": "constant", "value": 0.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 512, 16], + "data": { "kind": "fillFloat32", "sinStep": 0.11, "cosStep": 0.23, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 512, 16], + "data": { "kind": "fillFloat32", "sinStep": 0.07, "cosStep": 0.31, "scale": 0.5 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [511] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [512] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 64], "tolerance": 0.0005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 512, 16], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 512, 16], "tolerance": 0.0001 } + } + }, + { + "name": "qkv_flash_partial_subgroup_tile_seq23", + "provenance": { + "source": "onnxruntime/contrib_ops/webgpu/bert/flash_attention.wgsl.template", + "test": "partial-subgroup-tile masked-lane class", + "notes": "seq length 17-31 leaves the last subgroup tile partially populated; masked lanes must not contribute stale shuffle values to the online softmax." + }, + "attrs": { "num_heads": 12, "kv_num_heads": 4, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 23, 384], + "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.29, "scale": 0.5 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 23, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.11, "cosStep": 0.23, "scale": 0.5 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 23, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.07, "cosStep": 0.31, "scale": 0.5 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [22] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [23] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 23, 384], "tolerance": 0.0005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 4, 23, 32], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 4, 23, 32], "tolerance": 0.0001 } + } + }, + { + "name": "ort_share_append_decode_slack_b2q1cap8_h2kv1d8", + "attrs": { "num_heads": 2, "kv_num_heads": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [2, 1, 16], + "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.27, "scale": 0.5 } + }, + "keyT": { "dtype": "float32", "shape": [2, 1, 8], "data": { "kind": "linspace", "start": -1.0, "end": 1.0 } }, + "valueT": { "dtype": "float32", "shape": [2, 1, 8], "data": { "kind": "linspace", "start": 0.0, "end": 2.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [2, 1, 8, 8], + "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.31, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [2, 1, 8, 8], + "data": { "kind": "linspace", "start": 0.0, "end": 4.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [2], "data": { "kind": "values", "values": [3, 5] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [6] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [2, 1, 16], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [2, 1, 8, 8], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [2, 1, 8, 8], "tolerance": 0.0001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/webgpu/bert/group_query_attention.cc", + "test": "past_present_share_buffer append: CopyKVCache writes new tokens at seqlens_k[b]+1-newLen", + "notes": "Share-buffer append mode: present dims equal past dims (capacity); the new K/V rows are spliced at each batch's append offset and rows outside the append window mirror the past cache. seqlens_k[b]+1 is the post-append active end." + } + }, + { + "name": "ort_share_append_prompt_right_padded_b2q3cap8_h2kv1d8", + "attrs": { "num_heads": 2, "kv_num_heads": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [2, 3, 16], + "data": { "kind": "fillFloat32", "sinStep": 0.17, "cosStep": 0.23, "scale": 0.5 } + }, + "keyT": { + "dtype": "float32", + "shape": [2, 3, 8], + "data": { "kind": "fillFloat32", "sinStep": 0.29, "cosStep": 0.11, "scale": 0.5 } + }, + "valueT": { "dtype": "float32", "shape": [2, 3, 8], "data": { "kind": "linspace", "start": -1.0, "end": 3.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [2, 1, 8, 8], + "data": { "kind": "linspace", "start": 0.5, "end": 2.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [2, 1, 8, 8], + "data": { "kind": "linspace", "start": -2.0, "end": 2.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [2], "data": { "kind": "values", "values": [2, 1] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [3] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [2, 3, 16], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [2, 1, 8, 8], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [2, 1, 8, 8], "tolerance": 0.0001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/webgpu/bert/group_query_attention.cc", + "test": "past_present_share_buffer append: CopyKVCache writes new tokens at seqlens_k[b]+1-newLen", + "notes": "Share-buffer append mode: present dims equal past dims (capacity); the new K/V rows are spliced at each batch's append offset and rows outside the append window mirror the past cache. seqlens_k[b]+1 is the post-append active end." + } + }, + { + "name": "ort_share_append_chunk_prefill_b1q4cap16_h2kv1d8", + "attrs": { "num_heads": 2, "kv_num_heads": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 4, 16], + "data": { "kind": "fillFloat32", "sinStep": 0.21, "cosStep": 0.33, "scale": 0.5 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 4, 8], + "data": { "kind": "fillFloat32", "sinStep": 0.37, "cosStep": 0.13, "scale": 0.5 } + }, + "valueT": { "dtype": "float32", "shape": [1, 4, 8], "data": { "kind": "linspace", "start": 0.0, "end": 2.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 16, 8], + "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.31, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 16, 8], + "data": { "kind": "linspace", "start": 0.0, "end": 4.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [8] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [9] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 4, 16], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 16, 8], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 16, 8], "tolerance": 0.0001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/webgpu/bert/group_query_attention.cc", + "test": "past_present_share_buffer append: CopyKVCache writes new tokens at seqlens_k[b]+1-newLen", + "notes": "Share-buffer append mode: present dims equal past dims (capacity); the new K/V rows are spliced at each batch's append offset and rows outside the append window mirror the past cache. seqlens_k[b]+1 is the post-append active end." + } + }, + { + "name": "share_append_headsink_prefill_b1q10cap32_h4kv1d24", + "attrs": { "num_heads": 4, "kv_num_heads": 1, "scale": 0.20412414523193154, "local_window_size": 8 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 10, 96], + "data": { "kind": "fillFloat32", "sinStep": 0.11, "cosStep": 0.23, "scale": 0.5 } + }, + "keyT": { "dtype": "float32", "shape": [1, 10, 24], "data": { "kind": "linspace", "start": -1.0, "end": 1.0 } }, + "valueT": { "dtype": "float32", "shape": [1, 10, 24], "data": { "kind": "linspace", "start": 0.0, "end": 2.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 32, 24], + "data": { "kind": "fillFloat32", "sinStep": 0.17, "cosStep": 0.29, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 32, 24], + "data": { "kind": "linspace", "start": 0.0, "end": 4.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [9] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [10] } }, + "headSinkT": { "dtype": "float32", "shape": [4], "data": { "kind": "values", "values": [0.4, 0.0, -0.9, 1.1] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 10, 96], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 32, 24], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 32, 24], "tolerance": 0.0001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", + "test": "GQA head sink on the buffer-sharing append path", + "notes": "Exercises head-sink normalization while appending new K/V into a shared cache. The MQA geometry uses head dimension 24, capacity 32, a 10-token prompt, and local_window_size 8; zero and negative sink values make max/denominator renormalization observable." + } + }, + { + "name": "share_append_headsink_decode_slack_b1q1cap32_h4kv1d24", + "attrs": { "num_heads": 4, "kv_num_heads": 1, "scale": 0.20412414523193154, "local_window_size": 8 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 96], + "data": { "kind": "fillFloat32", "sinStep": 0.11, "cosStep": 0.23, "scale": 0.5 } + }, + "keyT": { "dtype": "float32", "shape": [1, 1, 24], "data": { "kind": "linspace", "start": -1.0, "end": 1.0 } }, + "valueT": { "dtype": "float32", "shape": [1, 1, 24], "data": { "kind": "linspace", "start": 0.0, "end": 2.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 32, 24], + "data": { "kind": "fillFloat32", "sinStep": 0.17, "cosStep": 0.29, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 32, 24], + "data": { "kind": "linspace", "start": 0.0, "end": 4.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [11] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [12] } }, + "headSinkT": { "dtype": "float32", "shape": [4], "data": { "kind": "values", "values": [0.4, 0.0, -0.9, 1.1] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 96], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 32, 24], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 32, 24], "tolerance": 0.0001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", + "test": "GQA head sink on the buffer-sharing append path", + "notes": "CASE: the decode step of the same geometry -- one new K/V row appended at 11 with 20 rows of capacity slack the attention bound must exclude, so seqlens_k rather than the bound capacity sets both the append offset and the window origin." + } + }, + { + "name": "share_append_decode_splitk_cap1024_b1q1_h2kv1d64", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.11, "cosStep": 0.29, "scale": 0.5 } + }, + "keyT": { "dtype": "float32", "shape": [1, 1, 64], "data": { "kind": "linspace", "start": -1.0, "end": 1.0 } }, + "valueT": { "dtype": "float32", "shape": [1, 1, 64], "data": { "kind": "linspace", "start": 0.0, "end": 2.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 1024, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.31, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 1024, 64], + "data": { "kind": "linspace", "start": 0.0, "end": 4.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [510] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [511] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 128], "tolerance": 0.001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 1024, 64], "tolerance": 0.001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 1024, 64], "tolerance": 0.001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/webgpu/bert/group_query_attention.cc", + "test": "past_present_share_buffer append: CopyKVCache writes new tokens at seqlens_k[b]+1-newLen", + "notes": "Share-buffer append mode: present dims equal past dims (capacity); the new K/V rows are spliced at each batch's append offset and rows outside the append window mirror the past cache. seqlens_k[b]+1 is the post-append active end." + } + }, + { + "name": "ort_share_append_rotary_decode_cap8_b1q1_h2kv1d16", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "do_rotary": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 32], + "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.27, "scale": 0.5 } + }, + "keyT": { "dtype": "float32", "shape": [1, 1, 16], "data": { "kind": "linspace", "start": -1.0, "end": 1.0 } }, + "valueT": { "dtype": "float32", "shape": [1, 1, 16], "data": { "kind": "linspace", "start": 0.0, "end": 2.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 8, 16], + "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.31, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 8, 16], + "data": { "kind": "linspace", "start": 0.0, "end": 4.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [3] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } }, + "cosCacheT": { + "dtype": "float32", + "shape": [8, 8], + "data": { "kind": "fillFloat32", "sinStep": 0.07, "cosStep": 0.19, "scale": 1.0 } + }, + "sinCacheT": { + "dtype": "float32", + "shape": [8, 8], + "data": { "kind": "fillFloat32", "sinStep": 0.23, "cosStep": 0.05, "scale": 1.0 } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 32], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 8, 16], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 8, 16], "tolerance": 0.0001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/webgpu/bert/group_query_attention.cc", + "test": "past_present_share_buffer append: CopyKVCache writes new tokens at seqlens_k[b]+1-newLen", + "notes": "Share-buffer append mode: present dims equal past dims (capacity); the new K/V rows are spliced at each batch's append offset and rows outside the append window mirror the past cache. seqlens_k[b]+1 is the post-append active end." + } + }, + { + "name": "prefill_tiled_q31_h8kv2_d36_non_cluster_stride", + "provenance": { + "notes": "Head size 36 is a multiple of 4 but not of 4*PREFILL_LANES_PER_QUERY=16, so the cooperative cluster prefill cannot map its per-query lanes; qSeq 31 clears the prefill minimum and qSeq*heads=248 clears the flash occupancy floor. Locks the blocked tiled prefill kernel on tiers without subgroups." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.16666666666666666, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 31, 288], + "data": { "kind": "fillFloat32", "scale": 0.08, "sinStep": 0.017, "cosStep": 0.031 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 32, 72], + "data": { "kind": "fillFloat32", "scale": 0.08, "sinStep": 0.011, "cosStep": 0.023 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 32, 72], + "data": { "kind": "fillFloat32", "scale": 0.08, "sinStep": 0.007, "cosStep": 0.041 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 31, 288], "tolerance": 0.0005, "relTolerance": 0.0005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 32, 36], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 32, 36], "tolerance": 0.000001 } + } + }, + { + "name": "flashprefill_softcap_h8kv2_d64_q32", + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125, "softcap": 15, "local_window_size": 16 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 32, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 32, 512], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 32, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 32, 64], "tolerance": 0.0001 } + }, + "provenance": { + "notes": "Logit soft-capping on the past_kv prefill path. Every flash prefill variant used to gate `standardSoftmax`, so a softcap sent the whole prefill to the scalar kernel at ~3% of peak; capping is a per-score transform, so the tiled kernel takes it unchanged." + } + }, + { + "name": "flashprefill_headsink_h8kv2_d64_q32", + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125, "local_window_size": 16 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 32, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } }, + "headSinkT": { + "dtype": "float32", + "shape": [8], + "data": { "kind": "values", "values": [0.4, -0.2, 1.1, 0.0, -0.9, 0.3, 0.7, -0.5] } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 32, 512], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 32, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 32, 64], "tolerance": 0.0001 } + }, + "provenance": { + "notes": "Head sink on the past_kv prefill path. The sink is a learned logit with no value attached, so it only enters the softmax denominator -- one term after the last tile, not a per-key change. Includes a zero and negative sinks so the max(m, sink) renormalization is exercised." + } + }, + { + "name": "flashprefill_bias_headsink_f16_h8kv2_d64_q32", + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125, "local_window_size": 16 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 32, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { "dtype": "float16", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float16", "shape": [1, 0, 128], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float16", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float16", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } }, + "attentionBiasT": { + "dtype": "float16", + "shape": [1, 8, 32, 32], + "data": { "kind": "fillFloat32", "scale": 0.1, "sinStep": 0.015, "cosStep": 0.021 } + }, + "headSinkT": { + "dtype": "float16", + "shape": [8], + "data": { "kind": "values", "values": [0.4, -0.2, 1.1, 0.0, -0.9, 0.3, 0.7, -0.5] } + } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 32, 512], "tolerance": 0.02 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 2, 32, 64], "tolerance": 0.001 }, + "presentValueT": { "dtype": "float16", "shape": [1, 2, 32, 64], "tolerance": 0.001 } + }, + "provenance": { + "notes": "attentionBias and head sink together on the past_kv prefill path: the bias rides the cluster kernel's additive-score path while the sink rides the denominator, and the two must compose." + } + }, + { + "name": "flashprefill_quant_int8_h4kv2_d64_q32", + "provenance": { + "notes": "INT8 KV cache on the tiled prefill path. The scalar kernel dequantized a cache element per (query, key); staging the tile dequantizes once per key and every query in the tile reads the result, so the unpack cost divides by the tile height." + }, + "attrs": { + "num_heads": 4, + "kv_num_heads": 2, + "kv_cache_bit_width": 8, + "scale": 0.125, + "k_quant_type": "PER_TENSOR", + "v_quant_type": "PER_TENSOR", + "local_window_size": 16 + }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 32, 256], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.029, "scale": 0.2 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 32, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.27, "cosStep": 0.18, "scale": 0.4 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 32, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.27, "cosStep": 0.18, "scale": 0.4 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } }, + "kScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [0.02] } }, + "vScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [0.02] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 32, 256], "tolerance": 0.01, "relTolerance": 0.01 }, + "presentKeyT": { "dtype": "int8", "shape": [1, 2, 32, 64], "tolerance": 1 }, + "presentValueT": { "dtype": "int8", "shape": [1, 2, 32, 64], "tolerance": 1 } + } + }, + { + "name": "flashprefill_quant_int4_h4kv2_d64_q32", + "provenance": { + "notes": "Packed INT4 KV cache on the tiled prefill path: two +8-biased signed nibbles per element, low nibble first, so a row is headDim/2 elements and the tile load reads two elements per staged vec4." + }, + "attrs": { + "num_heads": 4, + "kv_num_heads": 2, + "kv_cache_bit_width": 4, + "scale": 0.125, + "k_quant_type": "PER_TENSOR", + "v_quant_type": "PER_TENSOR", + "local_window_size": 16 + }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 32, 256], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.029, "scale": 0.2 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 32, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.27, "cosStep": 0.18, "scale": 0.4 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 32, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.27, "cosStep": 0.18, "scale": 0.4 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } }, + "kScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [0.02] } }, + "vScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [0.02] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 32, 256], "tolerance": 0.01, "relTolerance": 0.01 }, + "presentKeyT": { "dtype": "uint8", "shape": [1, 2, 32, 32], "tolerance": 1 }, + "presentValueT": { "dtype": "uint8", "shape": [1, 2, 32, 32], "tolerance": 1 } + } + }, + { + "name": "window_cache_no_eviction_b1q1cap8_h2kv1d8", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "sliding_window_cache": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 16], + "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.27, "scale": 0.5 } + }, + "keyT": { "dtype": "float32", "shape": [1, 1, 8], "data": { "kind": "linspace", "start": -1.0, "end": 1.0 } }, + "valueT": { "dtype": "float32", "shape": [1, 1, 8], "data": { "kind": "linspace", "start": 0.0, "end": 2.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.31, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "linspace", "start": 0.0, "end": 4.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [5] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [6] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 16], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", + "test": "GQA sliding_window_cache (ORT #29904), CPU/CUDA reference semantics", + "notes": "Windowed KV cache: the bound past/present buffer is a fixed CAPACITY holding only the most recent min(T, C) tokens contiguously at rows [0, L). Appending S tokens onto past P evicts E = max(0, min(P,C) + S - C) rows and slides survivors down by E. Attention reads keys at absolute positions translated by the cache origin T - L; RoPE keeps absolute positions. ORT's own WebGPU EP rejects this attribute (CUDA/CPU only). CASE: T=6 <= C=8, E=0, origin=0 — degenerates to append; also pins the unread-row clear." + } + }, + { + "name": "window_cache_decode_evicts_one_b1q1cap8_h2kv1d8", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "sliding_window_cache": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 16], + "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.27, "scale": 0.5 } + }, + "keyT": { "dtype": "float32", "shape": [1, 1, 8], "data": { "kind": "linspace", "start": -1.0, "end": 1.0 } }, + "valueT": { "dtype": "float32", "shape": [1, 1, 8], "data": { "kind": "linspace", "start": 0.0, "end": 2.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.31, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "linspace", "start": 0.0, "end": 4.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [11] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [12] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 16], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", + "test": "GQA sliding_window_cache (ORT #29904), CPU/CUDA reference semantics", + "notes": "Windowed KV cache: the bound past/present buffer is a fixed CAPACITY holding only the most recent min(T, C) tokens contiguously at rows [0, L). Appending S tokens onto past P evicts E = max(0, min(P,C) + S - C) rows and slides survivors down by E. Attention reads keys at absolute positions translated by the cache origin T - L; RoPE keeps absolute positions. ORT's own WebGPU EP rejects this attribute (CUDA/CPU only). CASE: T=12 > C=8, E=1, appendStart=7, origin=4 — the compaction shift." + } + }, + { + "name": "window_cache_chunk_evicts_two_b1q4cap8_h2kv1d8", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "sliding_window_cache": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 4, 16], + "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.27, "scale": 0.5 } + }, + "keyT": { "dtype": "float32", "shape": [1, 4, 8], "data": { "kind": "linspace", "start": -1.0, "end": 1.0 } }, + "valueT": { "dtype": "float32", "shape": [1, 4, 8], "data": { "kind": "linspace", "start": 0.0, "end": 2.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.31, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "linspace", "start": 0.0, "end": 4.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [9] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [10] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 4, 16], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", + "test": "GQA sliding_window_cache (ORT #29904), CPU/CUDA reference semantics", + "notes": "Windowed KV cache: the bound past/present buffer is a fixed CAPACITY holding only the most recent min(T, C) tokens contiguously at rows [0, L). Appending S tokens onto past P evicts E = max(0, min(P,C) + S - C) rows and slides survivors down by E. Attention reads keys at absolute positions translated by the cache origin T - L; RoPE keeps absolute positions. ORT's own WebGPU EP rejects this attribute (CUDA/CPU only). CASE: S=4, T=10 > C=8, E=2, appendStart=4, origin=2 — multi-row eviction." + } + }, + { + "name": "window_cache_mixed_batch_b2q1cap8_h2kv1d8", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "sliding_window_cache": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [2, 1, 16], + "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.27, "scale": 0.5 } + }, + "keyT": { "dtype": "float32", "shape": [2, 1, 8], "data": { "kind": "linspace", "start": -1.0, "end": 1.0 } }, + "valueT": { "dtype": "float32", "shape": [2, 1, 8], "data": { "kind": "linspace", "start": 0.0, "end": 2.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [2, 1, 8, 8], + "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.31, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [2, 1, 8, 8], + "data": { "kind": "linspace", "start": 0.0, "end": 4.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [2], "data": { "kind": "values", "values": [3, 11] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [12] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [2, 1, 16], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [2, 1, 8, 8], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [2, 1, 8, 8], "tolerance": 0.0001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", + "test": "GQA sliding_window_cache (ORT #29904), CPU/CUDA reference semantics", + "notes": "Windowed KV cache: the bound past/present buffer is a fixed CAPACITY holding only the most recent min(T, C) tokens contiguously at rows [0, L). Appending S tokens onto past P evicts E = max(0, min(P,C) + S - C) rows and slides survivors down by E. Attention reads keys at absolute positions translated by the cache origin T - L; RoPE keeps absolute positions. ORT's own WebGPU EP rejects this attribute (CUDA/CPU only). CASE: b0 T=4 (E=0, origin=0), b1 T=12 (E=1, origin=4) — per-batch origin." + } + }, + { + "name": "window_cache_localwindow_w4_b1q1cap8_h2kv1d8", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "sliding_window_cache": 1, "local_window_size": 4 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 16], + "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.27, "scale": 0.5 } + }, + "keyT": { "dtype": "float32", "shape": [1, 1, 8], "data": { "kind": "linspace", "start": -1.0, "end": 1.0 } }, + "valueT": { "dtype": "float32", "shape": [1, 1, 8], "data": { "kind": "linspace", "start": 0.0, "end": 2.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.31, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 8, 8], + "data": { "kind": "linspace", "start": 0.0, "end": 4.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [11] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [12] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 16], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 8, 8], "tolerance": 0.0001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", + "test": "GQA sliding_window_cache (ORT #29904), CPU/CUDA reference semantics", + "notes": "Windowed KV cache: the bound past/present buffer is a fixed CAPACITY holding only the most recent min(T, C) tokens contiguously at rows [0, L). Appending S tokens onto past P evicts E = max(0, min(P,C) + S - C) rows and slides survivors down by E. Attention reads keys at absolute positions translated by the cache origin T - L; RoPE keeps absolute positions. ORT's own WebGPU EP rejects this attribute (CUDA/CPU only). CASE: w=4 <= C=8 with E=1 — mask floor and cache origin interact." + } + }, + { + "name": "window_cache_capacity_equals_window_b1q1cap4_h2kv1d8", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "sliding_window_cache": 1, "local_window_size": 4 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 16], + "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.27, "scale": 0.5 } + }, + "keyT": { "dtype": "float32", "shape": [1, 1, 8], "data": { "kind": "linspace", "start": -1.0, "end": 1.0 } }, + "valueT": { "dtype": "float32", "shape": [1, 1, 8], "data": { "kind": "linspace", "start": 0.0, "end": 2.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 4, 8], + "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.31, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 4, 8], + "data": { "kind": "linspace", "start": 0.0, "end": 4.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [9] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [10] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 16], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 4, 8], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 4, 8], "tolerance": 0.0001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", + "test": "GQA sliding_window_cache (ORT #29904), CPU/CUDA reference semantics", + "notes": "Windowed KV cache: the bound past/present buffer is a fixed CAPACITY holding only the most recent min(T, C) tokens contiguously at rows [0, L). Appending S tokens onto past P evicts E = max(0, min(P,C) + S - C) rows and slides survivors down by E. Attention reads keys at absolute positions translated by the cache origin T - L; RoPE keeps absolute positions. ORT's own WebGPU EP rejects this attribute (CUDA/CPU only). CASE: C == w == 4, T=10 — the shape a sliding-window layer actually allocates." + } + }, + { + "name": "window_cache_gqa4_headdim16_b1q1cap16_h4kv1d16", + "attrs": { "num_heads": 4, "kv_num_heads": 1, "sliding_window_cache": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.27, "scale": 0.5 } + }, + "keyT": { "dtype": "float32", "shape": [1, 1, 16], "data": { "kind": "linspace", "start": -1.0, "end": 1.0 } }, + "valueT": { "dtype": "float32", "shape": [1, 1, 16], "data": { "kind": "linspace", "start": 0.0, "end": 2.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 16, 16], + "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.31, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 16, 16], + "data": { "kind": "linspace", "start": 0.0, "end": 4.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [63] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [64] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 64], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 16, 16], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 16, 16], "tolerance": 0.0001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", + "test": "GQA sliding_window_cache (ORT #29904), CPU/CUDA reference semantics", + "notes": "Windowed KV cache: the bound past/present buffer is a fixed CAPACITY holding only the most recent min(T, C) tokens contiguously at rows [0, L). Appending S tokens onto past P evicts E = max(0, min(P,C) + S - C) rows and slides survivors down by E. Attention reads keys at absolute positions translated by the cache origin T - L; RoPE keeps absolute positions. ORT's own WebGPU EP rejects this attribute (CUDA/CPU only). CASE: T=64 with C=16 — origin 48, far past the buffer." + } + }, + { + "name": "window_cache_decode_splitk_cap1024_b1q1_h2kv1d64", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "sliding_window_cache": 1, "local_window_size": 1024 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.27, "scale": 0.5 } + }, + "keyT": { "dtype": "float32", "shape": [1, 1, 64], "data": { "kind": "linspace", "start": -1.0, "end": 1.0 } }, + "valueT": { "dtype": "float32", "shape": [1, 1, 64], "data": { "kind": "linspace", "start": 0.0, "end": 2.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 1024, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.31, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 1024, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.23, "cosStep": 0.13, "scale": 0.5 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4095] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4096] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 128], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 1024, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 1024, 64], "tolerance": 0.0001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", + "test": "GQA sliding_window_cache (ORT #29904), CPU/CUDA reference semantics", + "notes": "Windowed cache at a capacity that clears CACHED_DECODE_MIN_KV_TOKENS, so the split-K decode variant is selected rather than the scalar fallback. T=4096 > C=1024, so the step genuinely evicts and the split-K kernel reads a shifted cache." + } + }, + { + "name": "window_cache_flash_chunk_evicts_b1q32cap64_h2kv1d64", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "sliding_window_cache": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 32, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.27, "scale": 0.5 } + }, + "keyT": { "dtype": "float32", "shape": [1, 32, 64], "data": { "kind": "linspace", "start": -1.0, "end": 1.0 } }, + "valueT": { "dtype": "float32", "shape": [1, 32, 64], "data": { "kind": "linspace", "start": 0.0, "end": 2.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 64, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.31, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 64, 64], + "data": { "kind": "linspace", "start": 0.0, "end": 4.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [79] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [80] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 32, 128], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 64, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 64, 64], "tolerance": 0.0001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", + "test": "GQA sliding_window_cache chunk prefill on the flash cluster kernel", + "notes": "Chunked flash prefill into a windowed cache: the present pass compacts entries, the attention bound comes from seqlens_k (kvActive), and the batch stride remains the cache capacity. T=80 > C=64, E=16, and kvActive=64 exercise eviction." + } + }, + { + "name": "window_cache_flash_chunk_unfilled_b1q32cap64_h2kv1d64", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "sliding_window_cache": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 32, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.27, "scale": 0.5 } + }, + "keyT": { "dtype": "float32", "shape": [1, 32, 64], "data": { "kind": "linspace", "start": -1.0, "end": 1.0 } }, + "valueT": { "dtype": "float32", "shape": [1, 32, 64], "data": { "kind": "linspace", "start": 0.0, "end": 2.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 64, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.31, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 64, 64], + "data": { "kind": "linspace", "start": 0.0, "end": 4.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [39] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [40] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 32, 128], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 64, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 64, 64], "tolerance": 0.0001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", + "test": "GQA sliding_window_cache chunk prefill on the flash cluster kernel", + "notes": "Chunked flash prefill into a windowed cache: the present pass compacts entries, the attention bound comes from seqlens_k (kvActive), and the batch stride remains the cache capacity. T=40 < C=64 makes kvActive smaller than capacity and distinguishes the bound from the stride." + } + }, + { + "name": "window_cache_flash_chunk_localwindow_w16_b1q32cap64_h2kv1d64", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "sliding_window_cache": 1, "local_window_size": 16 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 32, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.27, "scale": 0.5 } + }, + "keyT": { "dtype": "float32", "shape": [1, 32, 64], "data": { "kind": "linspace", "start": -1.0, "end": 1.0 } }, + "valueT": { "dtype": "float32", "shape": [1, 32, 64], "data": { "kind": "linspace", "start": 0.0, "end": 2.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 64, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.31, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 64, 64], + "data": { "kind": "linspace", "start": 0.0, "end": 4.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [79] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [80] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 32, 128], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 64, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 64, 64], "tolerance": 0.0001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", + "test": "GQA sliding_window_cache chunk prefill on the flash cluster kernel", + "notes": "Chunked flash prefill into a windowed cache: the present pass compacts entries, the attention bound comes from seqlens_k (kvActive), and the batch stride remains the cache capacity. A window of 16 below C=64 combines eviction with a nonzero window floor." + } + }, + { + "name": "window_cache_flash_chunk_mixed_batch_b2q32cap64_h2kv1d64", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "sliding_window_cache": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [2, 32, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.27, "scale": 0.5 } + }, + "keyT": { "dtype": "float32", "shape": [2, 32, 64], "data": { "kind": "linspace", "start": -1.0, "end": 1.0 } }, + "valueT": { "dtype": "float32", "shape": [2, 32, 64], "data": { "kind": "linspace", "start": 0.0, "end": 2.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [2, 1, 64, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.31, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [2, 1, 64, 64], + "data": { "kind": "linspace", "start": 0.0, "end": 4.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [2], "data": { "kind": "values", "values": [39, 79] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [80] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [2, 32, 128], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [2, 1, 64, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [2, 1, 64, 64], "tolerance": 0.0001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", + "test": "GQA sliding_window_cache chunk prefill on the flash cluster kernel", + "notes": "Chunked flash prefill into a windowed cache: the present pass compacts entries, the attention bound comes from seqlens_k (kvActive), and the batch stride remains the cache capacity. Batch 0 is unfilled at T=40 while batch 1 evicts at T=80, requiring kvActive to be tracked per batch." + } + }, + { + "name": "share_append_flash_chunk_slack_b1q32cap64_h2kv1d64", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "local_window_size": 16 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 32, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.11, "cosStep": 0.23, "scale": 0.5 } + }, + "keyT": { "dtype": "float32", "shape": [1, 32, 64], "data": { "kind": "linspace", "start": -1.0, "end": 1.0 } }, + "valueT": { "dtype": "float32", "shape": [1, 32, 64], "data": { "kind": "linspace", "start": 0.0, "end": 2.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 64, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.17, "cosStep": 0.29, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 64, 64], + "data": { "kind": "linspace", "start": 0.0, "end": 4.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [47] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [48] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 32, 128], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 64, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 64, 64], "tolerance": 0.0001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", + "test": "GQA buffer-sharing append chunk prefill on the flash cluster kernel", + "notes": "Share-append binds one fixed-capacity cache for past and present, so the resident length is seqlens_k + 1 rather than the bound capacity — the same split between attention bound and batch stride the windowed cache needs. headDim >= 64 with qSeq >= PREFILL_QUERY_TILE routes these to the flash prefill kernel. CASE: T=48 < C=64 — 16 rows of capacity slack the flash bound must exclude." + } + }, + { + "name": "share_append_flash_chunk_slack_b1q32cap64_h2kv1d256_register_boundary", + "attrs": { "num_heads": 2, "kv_num_heads": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 32, 512], + "data": { "kind": "fillFloat32", "sinStep": 0.11, "cosStep": 0.23, "scale": 0.5 } + }, + "keyT": { "dtype": "float32", "shape": [1, 32, 256], "data": { "kind": "linspace", "start": -1.0, "end": 1.0 } }, + "valueT": { + "dtype": "float32", + "shape": [1, 32, 256], + "data": { "kind": "linspace", "start": 0.0, "end": 2.0 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 64, 256], + "data": { "kind": "fillFloat32", "sinStep": 0.17, "cosStep": 0.29, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 64, 256], + "data": { "kind": "linspace", "start": 0.0, "end": 4.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [47] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [48] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 32, 512], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 64, 256], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 64, 256], "tolerance": 0.0001 } + }, + "provenance": { + "notes": "headDim-256 counterpart to the share-append capacity-slack case. It is above the f32 shared-memory cluster's register-geometry boundary and verifies consistent route admission for cached prefill while preserving the active-length versus capacity contract." + } + }, + { + "name": "share_append_flash_chunk_mixed_batch_b2q32cap64_h2kv1d64", + "attrs": { "num_heads": 2, "kv_num_heads": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [2, 32, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.11, "cosStep": 0.23, "scale": 0.5 } + }, + "keyT": { "dtype": "float32", "shape": [2, 32, 64], "data": { "kind": "linspace", "start": -1.0, "end": 1.0 } }, + "valueT": { "dtype": "float32", "shape": [2, 32, 64], "data": { "kind": "linspace", "start": 0.0, "end": 2.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [2, 1, 64, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.17, "cosStep": 0.29, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [2, 1, 64, 64], + "data": { "kind": "linspace", "start": 0.0, "end": 4.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [2], "data": { "kind": "values", "values": [31, 47] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [48] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [2, 32, 128], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [2, 1, 64, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [2, 1, 64, 64], "tolerance": 0.0001 } + }, + "provenance": { + "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", + "test": "GQA buffer-sharing append chunk prefill on the flash cluster kernel", + "notes": "Share-append binds one fixed-capacity cache for past and present, so the resident length is seqlens_k + 1 rather than the bound capacity — the same split between attention bound and batch stride the windowed cache needs. headDim >= 64 with qSeq >= PREFILL_QUERY_TILE routes these to the flash prefill kernel. CASE: b0 T=32 (append fills exactly), b1 T=48 — bound is per batch." + } + }, + { + "name": "splitk_decode_h8kv2_d64_kv2048_ramp_value_scale_lock", + "provenance": { + "notes": "Scale lock for present-KV split-K decode. A monotone V ramp keeps each output O(1) and dependent on the weighted key position, exposing errors in the softmax denominator, final divide, or cross-partition running-max rescale." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 512], + "data": { "kind": "fillFloat32", "scale": 0.18, "sinStep": 0.013, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 2048, 128], + "data": { "kind": "fillFloat32", "scale": 0.18, "sinStep": 0.019, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 2048, 128], + "data": { "kind": "linspace", "start": 0.5, "end": 2.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [2047] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [2048] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 512], "tolerance": 0.00005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 2048, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 2048, 64], "tolerance": 0.000001 } + } + }, + { + "name": "splitk_decode_h8kv2_d64_kv2048_f16_ramp_value_scale_lock", + "provenance": { + "notes": "The f16 compile of the present-KV decode split-K kernel had exactly one fixture, splitk_decode_h8kv2_d64_kv2048_f16, and its 0.03 absolute tolerance against a 2.1e-4 expected output is blind to a 146x scale error - the worst ratio in either attention op. Same shape and route with a ramped V so the f16 accumulator, its rescale and the final divide are all scale-locked." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 1, 512], + "data": { "kind": "fillFloat32", "scale": 0.18, "sinStep": 0.013, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float16", + "shape": [1, 2048, 128], + "data": { "kind": "fillFloat32", "scale": 0.18, "sinStep": 0.019, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float16", + "shape": [1, 2048, 128], + "data": { "kind": "linspace", "start": 0.5, "end": 2.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [2047] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [2048] } } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 1, 512], "tolerance": 0.003 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 2, 2048, 64], "tolerance": 0.001 }, + "presentValueT": { "dtype": "float16", "shape": [1, 2, 2048, 64], "tolerance": 0.001 } + } + }, + { + "name": "flash_prefill_h2kv1_d64_s128_f16_ramp_value_scale_lock", + "provenance": { + "notes": "qkv_present_flash_q32_broadcast and qkv_present_flash_q32_shared are reached by one fixture only, flash_prefill_h2kv1_d64_s128_f16, whose zero-mean V averages to 2.3e-3 under a 0.03 absolute tolerance - blind to a 13x scale error. Ramped V makes every causal row an O(1) function of its own weighted mean key index, so the register-blocked q32 epilogue divide is under test and each of the 128 query rows carries a different expected value." + }, + "attrs": { "num_heads": 2, "kv_num_heads": 1, "scale": 0.125, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 128, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.13, "cosStep": 0.29 } + }, + "keyT": { + "dtype": "float16", + "shape": [1, 128, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.19, "cosStep": 0.07 } + }, + "valueT": { + "dtype": "float16", + "shape": [1, 128, 64], + "data": { "kind": "linspace", "start": 0.5, "end": 2.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [127] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [128] } } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 128, 128], "tolerance": 0.003 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 1, 128, 64], "tolerance": 0.001 }, + "presentValueT": { "dtype": "float16", "shape": [1, 1, 128, 64], "tolerance": 0.001 } + } + }, + { + "name": "newkv_past_splitk_h8kv2_d64_q1p1024_ramp_value_scale_lock", + "provenance": { + "notes": "new_kv_past_decode_splitk and its _nosg twin - the append-new-KV-then-decode route - had one fixture, newkv_past_splitk_h8kv2_d64_q1p1024, whose zero-mean past and new V average to 4.7e-4 under a 0.005 tolerance: a 10x scale error passes. Ramping both value tensors makes the output an O(1) function of the weighted mean key index across the final 256 cached/new keys, so the local-window floor, split-K combine, softmax denominator, and seam between cached and new V are all scale-locked." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.013, "cosStep": 0.017 } + }, + "valueT": { "dtype": "float32", "shape": [1, 1, 128], "data": { "kind": "linspace", "start": 0.5, "end": 2.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 1024, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 1024, 64], + "data": { "kind": "linspace", "start": 0.5, "end": 2.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [1024] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [1025] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 512], "tolerance": 0.00005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 1025, 64], "tolerance": 0.001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 1025, 64], "tolerance": 0.001 } + } + }, + { + "name": "newkv_past_f16_flash_h8kv2_d64_q32p32_ramp_value_scale_lock", + "provenance": { + "notes": "new_kv_past_flash_prefill and its _nosg twin are reached only by newkv_past_flash_h8kv2_d64_q32p32 and its f16 sibling; the f16 one averages zero-mean V to 0.034 under a 0.03 tolerance, so an 89% scale error passes. Ramping both value tensors makes each of the 32 chunked-prefill rows an O(1) function of its 16-key causal local window; early rows cross the past/new seam, locking the window floor, flash normalization, and cache transition on the f16 compile." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125, "local_window_size": 16 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 32, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float16", + "shape": [1, 32, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.013, "cosStep": 0.017 } + }, + "valueT": { + "dtype": "float16", + "shape": [1, 32, 128], + "data": { "kind": "linspace", "start": 0.5, "end": 2.0 } + }, + "pastKeyT": { + "dtype": "float16", + "shape": [1, 2, 32, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float16", + "shape": [1, 2, 32, 64], + "data": { "kind": "linspace", "start": 0.5, "end": 2.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [63] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [64] } } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 32, 512], "tolerance": 0.003 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 2, 64, 64], "tolerance": 0.001 }, + "presentValueT": { "dtype": "float16", "shape": [1, 2, 64, 64], "tolerance": 0.001 } + } + }, + { + "name": "window_cache_decode_splitk_cap1024_b1q1_h2kv1d64_ramp_value_scale_lock", + "provenance": { + "notes": "window_shift_decode_splitk and its _nosg twin are reached by one fixture, window_cache_decode_splitk_cap1024_b1q1_h2kv1d64, which already ramps the one appended V but leaves the 1024-slot shifted cache zero-mean, so the average is 1.7e-3 under a 0.005 tolerance and a 3x scale error passes. Ramping the past cache as well makes the output an O(1) function of the weighted mean slot in the shifted window, so the split-K combine and the eviction offset both move it: T=4096 > C=1024, so the cache genuinely wraps." + }, + "attrs": { "num_heads": 2, "kv_num_heads": 1, "sliding_window_cache": 1, "local_window_size": 1024 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.27, "scale": 0.5 } + }, + "keyT": { "dtype": "float32", "shape": [1, 1, 64], "data": { "kind": "linspace", "start": -1.0, "end": 1.0 } }, + "valueT": { "dtype": "float32", "shape": [1, 1, 64], "data": { "kind": "linspace", "start": 0.5, "end": 2.0 } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 1, 1024, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.19, "cosStep": 0.31, "scale": 0.5 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 1024, 64], + "data": { "kind": "linspace", "start": 0.5, "end": 2.0 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4095] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4096] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 128], "tolerance": 0.00005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 1024, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 1024, 64], "tolerance": 0.0001 } + } + }, + { + "name": "quant_int8_scalar_prompt_ramp_value_scale_lock", + "provenance": { + "notes": "The scalar INT8 KV-cache route is reached only by ORT-derived fixtures carrying a 0.05 absolute tolerance against an O(0.5) output, so a 10% uniform scale error passes every one of them; unlike the other attention holes this is loose tolerance rather than a cancelled average. A synthetic sibling on the same route reproduces the reference to under 1e-7 - the kernel and the oracle quantize identically, so the ORT tolerance was never needed here - which locks the vScale dequant, the softmax denominator and the apply divide at 2e-5. V ramps 0.5 to 2.0 so each causal row lands on a different O(1) value." + }, + "attrs": { + "num_heads": 2, + "kv_num_heads": 1, + "kv_cache_bit_width": 8, + "k_quant_type": "PER_TENSOR", + "v_quant_type": "PER_TENSOR" + }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 4, 16], + "data": { "kind": "fillFloat32", "scale": 0.3, "sinStep": 0.13, "cosStep": 0.29 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 4, 8], + "data": { "kind": "fillFloat32", "scale": 0.3, "sinStep": 0.19, "cosStep": 0.07 } + }, + "valueT": { "dtype": "float32", "shape": [1, 4, 8], "data": { "kind": "linspace", "start": 0.5, "end": 2.0 } }, + "pastKeyT": { "dtype": "int8", "shape": [1, 1, 4, 8], "data": { "kind": "constant", "value": 0 } }, + "pastValueT": { "dtype": "int8", "shape": [1, 1, 4, 8], "data": { "kind": "constant", "value": 0 } }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [3] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } }, + "kScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [0.003] } }, + "vScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [0.02] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 4, 16], "tolerance": 0.00002 }, + "presentKeyT": { "dtype": "int8", "shape": [1, 1, 4, 8], "tolerance": 0 }, + "presentValueT": { "dtype": "int8", "shape": [1, 1, 4, 8], "tolerance": 0 } + } + }, + { + "name": "quant_int4_scalar_prompt_ramp_value_scale_lock", + "provenance": { + "notes": "Same hole as the INT8 sibling but wider: the scalar INT4 KV-cache route only has ORT-derived fixtures at a 0.15 absolute tolerance over an O(0.5) output, blind to a 30% scale error. vScale 0.3 spans the ramp inside the signed 4-bit range, and the kernel matches the oracle to under 1e-7, so a 2e-5 tolerance locks the +8-biased nibble unpack, the dequant scale and the normalization." + }, + "attrs": { + "num_heads": 2, + "kv_num_heads": 1, + "kv_cache_bit_width": 4, + "k_quant_type": "PER_TENSOR", + "v_quant_type": "PER_TENSOR" + }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 4, 16], + "data": { "kind": "fillFloat32", "scale": 0.3, "sinStep": 0.13, "cosStep": 0.29 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 4, 8], + "data": { "kind": "fillFloat32", "scale": 0.3, "sinStep": 0.19, "cosStep": 0.07 } + }, + "valueT": { "dtype": "float32", "shape": [1, 4, 8], "data": { "kind": "linspace", "start": 0.5, "end": 2.0 } }, + "pastKeyT": { "dtype": "uint8", "shape": [1, 1, 4, 4], "data": { "kind": "constant", "value": 0 } }, + "pastValueT": { "dtype": "uint8", "shape": [1, 1, 4, 4], "data": { "kind": "constant", "value": 0 } }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [3] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } }, + "kScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [0.05] } }, + "vScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [0.3] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 4, 16], "tolerance": 0.00002 }, + "presentKeyT": { "dtype": "uint8", "shape": [1, 1, 4, 4], "tolerance": 0 }, + "presentValueT": { "dtype": "uint8", "shape": [1, 1, 4, 4], "tolerance": 0 } + } + } + ] +}