| { |
| "op": "com.microsoft.MatMulNBitsQkv", |
| "fixtureArrays": { |
| "norm_gqa_input_vBT": [90, 17, 203, 156, 64, 241, 112, 38, 175, 229, 83, 11, 198, 147, 52, 220, 105] |
| }, |
| "cases": [ |
| { |
| "name": "norm_gqa", |
| "provenance": { |
| "notes": "Grouped-query shape: Nq is twice Nkv, so the three projections have two different column counts." |
| }, |
| "attrs": { "K": 32, "Nq": 8, "Nkv": 4, "block_size": 32 }, |
| "inputs": { |
| "aT": { |
| "dtype": "float32", |
| "shape": [3, 32], |
| "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.29, "scale": 0.8 } |
| }, |
| "normScaleT": { |
| "dtype": "float32", |
| "shape": [32], |
| "data": { "kind": "fillFloat32", "sinStep": 0.23, "cosStep": 0.41, "scale": 0.4, "offset": 1.0 } |
| }, |
| "qBT": { |
| "dtype": "uint8", |
| "shape": [8, 1, 16], |
| "data": { "kind": "cycle", "values": [27, 180, 75, 226, 33, 150, 201, 108, 57, 246, 129, 66, 195] } |
| }, |
| "qScalesT": { |
| "dtype": "float32", |
| "shape": [8, 1], |
| "data": { "kind": "cycle", "values": [0.04, 0.055, 0.05, 0.065, 0.06, 0.075, 0.07] } |
| }, |
| "kBT": { |
| "dtype": "uint8", |
| "shape": [4, 1, 16], |
| "data": { "kind": "cycle", "values": [211, 44, 137, 98, 165, 20, 233, 121, 78, 190, 15, 252, 87, 143, 61] } |
| }, |
| "kScalesT": { |
| "dtype": "float32", |
| "shape": [4, 1], |
| "data": { "kind": "cycle", "values": [0.045, 0.03, 0.07, 0.05, 0.08, 0.035] } |
| }, |
| "vBT": { |
| "dtype": "uint8", |
| "shape": [4, 1, 16], |
| "data": { "kind": "cycle", "values": { "$ref": "#/fixtureArrays/norm_gqa_input_vBT" } } |
| }, |
| "vScalesT": { |
| "dtype": "float32", |
| "shape": [4, 1], |
| "data": { "kind": "cycle", "values": [0.052, 0.038, 0.061, 0.029, 0.073] } |
| } |
| }, |
| "outputs": { |
| "qT": { "dtype": "float32", "shape": [3, 8], "tolerance": 0.0001, "relTolerance": 0.0001 }, |
| "kT": { "dtype": "float32", "shape": [3, 4], "tolerance": 0.0001, "relTolerance": 0.0001 }, |
| "vT": { "dtype": "float32", "shape": [3, 4], "tolerance": 0.0001, "relTolerance": 0.0001 } |
| } |
| }, |
| { |
| "name": "norm_rows_past_one_tile", |
| "provenance": { |
| "notes": "Ten activation rows against an eight-row projection tile: two row groups where the second carries only two real rows, so the store guard has to drop the six that clamped onto the last row." |
| }, |
| "attrs": { "K": 32, "Nq": 8, "Nkv": 4, "block_size": 32 }, |
| "inputs": { |
| "aT": { |
| "dtype": "float32", |
| "shape": [10, 32], |
| "data": { "kind": "fillFloat32", "sinStep": 0.17, "cosStep": 0.31, "scale": 0.8 } |
| }, |
| "normScaleT": { |
| "dtype": "float32", |
| "shape": [32], |
| "data": { "kind": "fillFloat32", "sinStep": 0.23, "cosStep": 0.41, "scale": 0.4, "offset": 1.0 } |
| }, |
| "qBT": { |
| "dtype": "uint8", |
| "shape": [8, 1, 16], |
| "data": { "kind": "cycle", "values": [27, 180, 75, 226, 33, 150, 201, 108, 57, 246, 129, 66, 195] } |
| }, |
| "qScalesT": { |
| "dtype": "float32", |
| "shape": [8, 1], |
| "data": { "kind": "cycle", "values": [0.04, 0.055, 0.05, 0.065, 0.06, 0.075, 0.07] } |
| }, |
| "kBT": { |
| "dtype": "uint8", |
| "shape": [4, 1, 16], |
| "data": { "kind": "cycle", "values": [211, 44, 137, 98, 165, 20, 233, 121, 78, 190, 15, 252, 87, 143, 61] } |
| }, |
| "kScalesT": { |
| "dtype": "float32", |
| "shape": [4, 1], |
| "data": { "kind": "cycle", "values": [0.045, 0.03, 0.07, 0.05, 0.08, 0.035] } |
| }, |
| "vBT": { |
| "dtype": "uint8", |
| "shape": [4, 1, 16], |
| "data": { "kind": "cycle", "values": { "$ref": "#/fixtureArrays/norm_gqa_input_vBT" } } |
| }, |
| "vScalesT": { |
| "dtype": "float32", |
| "shape": [4, 1], |
| "data": { "kind": "cycle", "values": [0.052, 0.038, 0.061, 0.029, 0.073] } |
| } |
| }, |
| "outputs": { |
| "qT": { "dtype": "float32", "shape": [10, 8], "tolerance": 0.0001, "relTolerance": 0.0001 }, |
| "kT": { "dtype": "float32", "shape": [10, 4], "tolerance": 0.0001, "relTolerance": 0.0001 }, |
| "vT": { "dtype": "float32", "shape": [10, 4], "tolerance": 0.0001, "relTolerance": 0.0001 } |
| } |
| }, |
| { |
| "name": "skip_gqa", |
| "attrs": { "K": 32, "Nq": 8, "Nkv": 4, "block_size": 32, "epsilon": 0.00001 }, |
| "inputs": { |
| "aT": { |
| "dtype": "float32", |
| "shape": [3, 32], |
| "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.29, "scale": 0.8 } |
| }, |
| "skipT": { |
| "dtype": "float32", |
| "shape": [3, 32], |
| "data": { "kind": "fillFloat32", "sinStep": 0.37, "cosStep": 0.11, "scale": 0.5 } |
| }, |
| "normScaleT": { |
| "dtype": "float32", |
| "shape": [32], |
| "data": { "kind": "fillFloat32", "sinStep": 0.23, "cosStep": 0.41, "scale": 0.4, "offset": 1.0 } |
| }, |
| "qBT": { |
| "dtype": "uint8", |
| "shape": [8, 1, 16], |
| "data": { "kind": "cycle", "values": [27, 180, 75, 226, 33, 150, 201, 108, 57, 246, 129, 66, 195] } |
| }, |
| "qScalesT": { |
| "dtype": "float32", |
| "shape": [8, 1], |
| "data": { "kind": "cycle", "values": [0.04, 0.055, 0.05, 0.065, 0.06, 0.075, 0.07] } |
| }, |
| "kBT": { |
| "dtype": "uint8", |
| "shape": [4, 1, 16], |
| "data": { "kind": "cycle", "values": [211, 44, 137, 98, 165, 20, 233, 121, 78, 190, 15, 252, 87, 143, 61] } |
| }, |
| "kScalesT": { |
| "dtype": "float32", |
| "shape": [4, 1], |
| "data": { "kind": "cycle", "values": [0.045, 0.03, 0.07, 0.05, 0.08, 0.035] } |
| }, |
| "vBT": { |
| "dtype": "uint8", |
| "shape": [4, 1, 16], |
| "data": { "kind": "cycle", "values": { "$ref": "#/fixtureArrays/norm_gqa_input_vBT" } } |
| }, |
| "vScalesT": { |
| "dtype": "float32", |
| "shape": [4, 1], |
| "data": { "kind": "cycle", "values": [0.052, 0.038, 0.061, 0.029, 0.073] } |
| } |
| }, |
| "outputs": { |
| "qT": { "dtype": "float32", "shape": [3, 8], "tolerance": 0.0001, "relTolerance": 0.0001 }, |
| "kT": { "dtype": "float32", "shape": [3, 4], "tolerance": 0.0001, "relTolerance": 0.0001 }, |
| "vT": { "dtype": "float32", "shape": [3, 4], "tolerance": 0.0001, "relTolerance": 0.0001 } |
| } |
| }, |
| { |
| "name": "skipsum_gqa", |
| "attrs": { "K": 32, "Nq": 8, "Nkv": 4, "block_size": 32 }, |
| "inputs": { |
| "aT": { |
| "dtype": "float32", |
| "shape": [3, 32], |
| "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.29, "scale": 0.8 } |
| }, |
| "skipT": { |
| "dtype": "float32", |
| "shape": [3, 32], |
| "data": { "kind": "fillFloat32", "sinStep": 0.37, "cosStep": 0.11, "scale": 0.5 } |
| }, |
| "normScaleT": { |
| "dtype": "float32", |
| "shape": [32], |
| "data": { "kind": "fillFloat32", "sinStep": 0.23, "cosStep": 0.41, "scale": 0.4, "offset": 1.0 } |
| }, |
| "qBT": { |
| "dtype": "uint8", |
| "shape": [8, 1, 16], |
| "data": { "kind": "cycle", "values": [27, 180, 75, 226, 33, 150, 201, 108, 57, 246, 129, 66, 195] } |
| }, |
| "qScalesT": { |
| "dtype": "float32", |
| "shape": [8, 1], |
| "data": { "kind": "cycle", "values": [0.04, 0.055, 0.05, 0.065, 0.06, 0.075, 0.07] } |
| }, |
| "kBT": { |
| "dtype": "uint8", |
| "shape": [4, 1, 16], |
| "data": { "kind": "cycle", "values": [211, 44, 137, 98, 165, 20, 233, 121, 78, 190, 15, 252, 87, 143, 61] } |
| }, |
| "kScalesT": { |
| "dtype": "float32", |
| "shape": [4, 1], |
| "data": { "kind": "cycle", "values": [0.045, 0.03, 0.07, 0.05, 0.08, 0.035] } |
| }, |
| "vBT": { |
| "dtype": "uint8", |
| "shape": [4, 1, 16], |
| "data": { "kind": "cycle", "values": { "$ref": "#/fixtureArrays/norm_gqa_input_vBT" } } |
| }, |
| "vScalesT": { |
| "dtype": "float32", |
| "shape": [4, 1], |
| "data": { "kind": "cycle", "values": [0.052, 0.038, 0.061, 0.029, 0.073] } |
| } |
| }, |
| "outputs": { |
| "qT": { "dtype": "float32", "shape": [3, 8], "tolerance": 0.0001, "relTolerance": 0.0001 }, |
| "kT": { "dtype": "float32", "shape": [3, 4], "tolerance": 0.0001, "relTolerance": 0.0001 }, |
| "vT": { "dtype": "float32", "shape": [3, 4], "tolerance": 0.0001, "relTolerance": 0.0001 }, |
| "residualT": { "dtype": "float32", "shape": [3, 32], "tolerance": 0.000001, "relTolerance": 0.000001 } |
| } |
| }, |
| { |
| "name": "decode_single_row", |
| "attrs": { "K": 32, "Nq": 8, "Nkv": 4, "block_size": 32 }, |
| "inputs": { |
| "aT": { |
| "dtype": "float32", |
| "shape": [1, 32], |
| "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.29, "scale": 0.8 } |
| }, |
| "skipT": { |
| "dtype": "float32", |
| "shape": [1, 32], |
| "data": { "kind": "fillFloat32", "sinStep": 0.37, "cosStep": 0.11, "scale": 0.5 } |
| }, |
| "normScaleT": { |
| "dtype": "float32", |
| "shape": [32], |
| "data": { "kind": "fillFloat32", "sinStep": 0.23, "cosStep": 0.41, "scale": 0.4, "offset": 1.0 } |
| }, |
| "qBT": { |
| "dtype": "uint8", |
| "shape": [8, 1, 16], |
| "data": { "kind": "cycle", "values": [27, 180, 75, 226, 33, 150, 201, 108, 57, 246, 129, 66, 195] } |
| }, |
| "qScalesT": { |
| "dtype": "float32", |
| "shape": [8, 1], |
| "data": { "kind": "cycle", "values": [0.04, 0.055, 0.05, 0.065, 0.06, 0.075, 0.07] } |
| }, |
| "kBT": { |
| "dtype": "uint8", |
| "shape": [4, 1, 16], |
| "data": { "kind": "cycle", "values": [211, 44, 137, 98, 165, 20, 233, 121, 78, 190, 15, 252, 87, 143, 61] } |
| }, |
| "kScalesT": { |
| "dtype": "float32", |
| "shape": [4, 1], |
| "data": { "kind": "cycle", "values": [0.045, 0.03, 0.07, 0.05, 0.08, 0.035] } |
| }, |
| "vBT": { |
| "dtype": "uint8", |
| "shape": [4, 1, 16], |
| "data": { "kind": "cycle", "values": { "$ref": "#/fixtureArrays/norm_gqa_input_vBT" } } |
| }, |
| "vScalesT": { |
| "dtype": "float32", |
| "shape": [4, 1], |
| "data": { "kind": "cycle", "values": [0.052, 0.038, 0.061, 0.029, 0.073] } |
| } |
| }, |
| "outputs": { |
| "qT": { "dtype": "float32", "shape": [1, 8], "tolerance": 0.0001, "relTolerance": 0.0001 }, |
| "kT": { "dtype": "float32", "shape": [1, 4], "tolerance": 0.0001, "relTolerance": 0.0001 }, |
| "vT": { "dtype": "float32", "shape": [1, 4], "tolerance": 0.0001, "relTolerance": 0.0001 }, |
| "residualT": { "dtype": "float32", "shape": [1, 32], "tolerance": 0.000001, "relTolerance": 0.000001 } |
| } |
| }, |
| { |
| "name": "equal_nq_nkv", |
| "provenance": { |
| "notes": "Multi-head shape where Nq equals Nkv; the tile grid still lays Q, K and V out in sequence." |
| }, |
| "attrs": { "K": 32, "Nq": 8, "Nkv": 8, "block_size": 32 }, |
| "inputs": { |
| "aT": { |
| "dtype": "float32", |
| "shape": [2, 32], |
| "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.29, "scale": 0.8 } |
| }, |
| "normScaleT": { |
| "dtype": "float32", |
| "shape": [32], |
| "data": { "kind": "fillFloat32", "sinStep": 0.23, "cosStep": 0.41, "scale": 0.4, "offset": 1.0 } |
| }, |
| "qBT": { |
| "dtype": "uint8", |
| "shape": [8, 1, 16], |
| "data": { "kind": "cycle", "values": [27, 180, 75, 226, 33, 150, 201, 108, 57, 246, 129, 66, 195] } |
| }, |
| "qScalesT": { |
| "dtype": "float32", |
| "shape": [8, 1], |
| "data": { "kind": "cycle", "values": [0.04, 0.055, 0.05, 0.065, 0.06, 0.075, 0.07] } |
| }, |
| "kBT": { |
| "dtype": "uint8", |
| "shape": [8, 1, 16], |
| "data": { "kind": "cycle", "values": [211, 44, 137, 98, 165, 20, 233, 121, 78, 190, 15, 252, 87, 143, 61] } |
| }, |
| "kScalesT": { |
| "dtype": "float32", |
| "shape": [8, 1], |
| "data": { "kind": "cycle", "values": [0.045, 0.03, 0.07, 0.05, 0.08, 0.035] } |
| }, |
| "vBT": { |
| "dtype": "uint8", |
| "shape": [8, 1, 16], |
| "data": { "kind": "cycle", "values": { "$ref": "#/fixtureArrays/norm_gqa_input_vBT" } } |
| }, |
| "vScalesT": { |
| "dtype": "float32", |
| "shape": [8, 1], |
| "data": { "kind": "cycle", "values": [0.052, 0.038, 0.061, 0.029, 0.073] } |
| } |
| }, |
| "outputs": { |
| "qT": { "dtype": "float32", "shape": [2, 8], "tolerance": 0.0001, "relTolerance": 0.0001 }, |
| "kT": { "dtype": "float32", "shape": [2, 8], "tolerance": 0.0001, "relTolerance": 0.0001 }, |
| "vT": { "dtype": "float32", "shape": [2, 8], "tolerance": 0.0001, "relTolerance": 0.0001 } |
| } |
| }, |
| { |
| "name": "tiles_not_aligned", |
| "provenance": { |
| "notes": "Neither column count is a multiple of the column tile, so both projections have a partial last tile." |
| }, |
| "attrs": { "K": 32, "Nq": 5, "Nkv": 3, "block_size": 32 }, |
| "inputs": { |
| "aT": { |
| "dtype": "float32", |
| "shape": [2, 32], |
| "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.29, "scale": 0.8 } |
| }, |
| "normScaleT": { |
| "dtype": "float32", |
| "shape": [32], |
| "data": { "kind": "fillFloat32", "sinStep": 0.23, "cosStep": 0.41, "scale": 0.4, "offset": 1.0 } |
| }, |
| "qBT": { |
| "dtype": "uint8", |
| "shape": [5, 1, 16], |
| "data": { "kind": "cycle", "values": [27, 180, 75, 226, 33, 150, 201, 108, 57, 246, 129, 66, 195] } |
| }, |
| "qScalesT": { |
| "dtype": "float32", |
| "shape": [5, 1], |
| "data": { "kind": "cycle", "values": [0.04, 0.055, 0.05, 0.065, 0.06, 0.075, 0.07] } |
| }, |
| "kBT": { |
| "dtype": "uint8", |
| "shape": [3, 1, 16], |
| "data": { "kind": "cycle", "values": [211, 44, 137, 98, 165, 20, 233, 121, 78, 190, 15, 252, 87, 143, 61] } |
| }, |
| "kScalesT": { |
| "dtype": "float32", |
| "shape": [3, 1], |
| "data": { "kind": "cycle", "values": [0.045, 0.03, 0.07, 0.05, 0.08, 0.035] } |
| }, |
| "vBT": { |
| "dtype": "uint8", |
| "shape": [3, 1, 16], |
| "data": { "kind": "cycle", "values": { "$ref": "#/fixtureArrays/norm_gqa_input_vBT" } } |
| }, |
| "vScalesT": { |
| "dtype": "float32", |
| "shape": [3, 1], |
| "data": { "kind": "cycle", "values": [0.052, 0.038, 0.061, 0.029, 0.073] } |
| } |
| }, |
| "outputs": { |
| "qT": { "dtype": "float32", "shape": [2, 5], "tolerance": 0.0001, "relTolerance": 0.0001 }, |
| "kT": { "dtype": "float32", "shape": [2, 3], "tolerance": 0.0001, "relTolerance": 0.0001 }, |
| "vT": { "dtype": "float32", "shape": [2, 3], "tolerance": 0.0001, "relTolerance": 0.0001 } |
| } |
| }, |
| { |
| "name": "tail_block_partial_k", |
| "provenance": { "notes": "K is not a multiple of block_size, so the final block is partly out of range." }, |
| "attrs": { "K": 20, "Nq": 8, "Nkv": 4, "block_size": 32 }, |
| "inputs": { |
| "aT": { |
| "dtype": "float32", |
| "shape": [2, 20], |
| "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.29, "scale": 0.8 } |
| }, |
| "normScaleT": { |
| "dtype": "float32", |
| "shape": [20], |
| "data": { "kind": "fillFloat32", "sinStep": 0.23, "cosStep": 0.41, "scale": 0.4, "offset": 1.0 } |
| }, |
| "qBT": { |
| "dtype": "uint8", |
| "shape": [8, 1, 16], |
| "data": { "kind": "cycle", "values": [27, 180, 75, 226, 33, 150, 201, 108, 57, 246, 129, 66, 195] } |
| }, |
| "qScalesT": { |
| "dtype": "float32", |
| "shape": [8, 1], |
| "data": { "kind": "cycle", "values": [0.04, 0.055, 0.05, 0.065, 0.06, 0.075, 0.07] } |
| }, |
| "kBT": { |
| "dtype": "uint8", |
| "shape": [4, 1, 16], |
| "data": { "kind": "cycle", "values": [211, 44, 137, 98, 165, 20, 233, 121, 78, 190, 15, 252, 87, 143, 61] } |
| }, |
| "kScalesT": { |
| "dtype": "float32", |
| "shape": [4, 1], |
| "data": { "kind": "cycle", "values": [0.045, 0.03, 0.07, 0.05, 0.08, 0.035] } |
| }, |
| "vBT": { |
| "dtype": "uint8", |
| "shape": [4, 1, 16], |
| "data": { "kind": "cycle", "values": { "$ref": "#/fixtureArrays/norm_gqa_input_vBT" } } |
| }, |
| "vScalesT": { |
| "dtype": "float32", |
| "shape": [4, 1], |
| "data": { "kind": "cycle", "values": [0.052, 0.038, 0.061, 0.029, 0.073] } |
| } |
| }, |
| "outputs": { |
| "qT": { "dtype": "float32", "shape": [2, 8], "tolerance": 0.0001, "relTolerance": 0.0001 }, |
| "kT": { "dtype": "float32", "shape": [2, 4], "tolerance": 0.0001, "relTolerance": 0.0001 }, |
| "vT": { "dtype": "float32", "shape": [2, 4], "tolerance": 0.0001, "relTolerance": 0.0001 } |
| } |
| }, |
| { |
| "name": "rank3_activation", |
| "attrs": { "K": 32, "Nq": 8, "Nkv": 4, "block_size": 32 }, |
| "inputs": { |
| "aT": { |
| "dtype": "float32", |
| "shape": [1, 4, 32], |
| "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.29, "scale": 0.8 } |
| }, |
| "normScaleT": { |
| "dtype": "float32", |
| "shape": [32], |
| "data": { "kind": "fillFloat32", "sinStep": 0.23, "cosStep": 0.41, "scale": 0.4, "offset": 1.0 } |
| }, |
| "qBT": { |
| "dtype": "uint8", |
| "shape": [8, 1, 16], |
| "data": { "kind": "cycle", "values": [27, 180, 75, 226, 33, 150, 201, 108, 57, 246, 129, 66, 195] } |
| }, |
| "qScalesT": { |
| "dtype": "float32", |
| "shape": [8, 1], |
| "data": { "kind": "cycle", "values": [0.04, 0.055, 0.05, 0.065, 0.06, 0.075, 0.07] } |
| }, |
| "kBT": { |
| "dtype": "uint8", |
| "shape": [4, 1, 16], |
| "data": { "kind": "cycle", "values": [211, 44, 137, 98, 165, 20, 233, 121, 78, 190, 15, 252, 87, 143, 61] } |
| }, |
| "kScalesT": { |
| "dtype": "float32", |
| "shape": [4, 1], |
| "data": { "kind": "cycle", "values": [0.045, 0.03, 0.07, 0.05, 0.08, 0.035] } |
| }, |
| "vBT": { |
| "dtype": "uint8", |
| "shape": [4, 1, 16], |
| "data": { "kind": "cycle", "values": { "$ref": "#/fixtureArrays/norm_gqa_input_vBT" } } |
| }, |
| "vScalesT": { |
| "dtype": "float32", |
| "shape": [4, 1], |
| "data": { "kind": "cycle", "values": [0.052, 0.038, 0.061, 0.029, 0.073] } |
| } |
| }, |
| "outputs": { |
| "qT": { "dtype": "float32", "shape": [1, 4, 8], "tolerance": 0.0001, "relTolerance": 0.0001 }, |
| "kT": { "dtype": "float32", "shape": [1, 4, 4], "tolerance": 0.0001, "relTolerance": 0.0001 }, |
| "vT": { "dtype": "float32", "shape": [1, 4, 4], "tolerance": 0.0001, "relTolerance": 0.0001 } |
| } |
| }, |
| { |
| "name": "two_quant_blocks", |
| "provenance": { "notes": "K=64 covers two of the provider's supported 32-element quantization blocks." }, |
| "attrs": { "K": 64, "Nq": 8, "Nkv": 4, "block_size": 32 }, |
| "inputs": { |
| "aT": { |
| "dtype": "float32", |
| "shape": [2, 64], |
| "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.29, "scale": 0.8 } |
| }, |
| "skipT": { |
| "dtype": "float32", |
| "shape": [2, 64], |
| "data": { "kind": "fillFloat32", "sinStep": 0.37, "cosStep": 0.11, "scale": 0.5 } |
| }, |
| "normScaleT": { |
| "dtype": "float32", |
| "shape": [64], |
| "data": { "kind": "fillFloat32", "sinStep": 0.23, "cosStep": 0.41, "scale": 0.4, "offset": 1.0 } |
| }, |
| "qBT": { |
| "dtype": "uint8", |
| "shape": [8, 2, 16], |
| "data": { "kind": "cycle", "values": [27, 180, 75, 226, 33, 150, 201, 108, 57, 246, 129, 66, 195] } |
| }, |
| "qScalesT": { |
| "dtype": "float32", |
| "shape": [8, 2], |
| "data": { "kind": "cycle", "values": [0.04, 0.055, 0.05, 0.065, 0.06, 0.075, 0.07] } |
| }, |
| "kBT": { |
| "dtype": "uint8", |
| "shape": [4, 2, 16], |
| "data": { "kind": "cycle", "values": [211, 44, 137, 98, 165, 20, 233, 121, 78, 190, 15, 252, 87, 143, 61] } |
| }, |
| "kScalesT": { |
| "dtype": "float32", |
| "shape": [4, 2], |
| "data": { "kind": "cycle", "values": [0.045, 0.03, 0.07, 0.05, 0.08, 0.035] } |
| }, |
| "vBT": { |
| "dtype": "uint8", |
| "shape": [4, 2, 16], |
| "data": { "kind": "cycle", "values": { "$ref": "#/fixtureArrays/norm_gqa_input_vBT" } } |
| }, |
| "vScalesT": { |
| "dtype": "float32", |
| "shape": [4, 2], |
| "data": { "kind": "cycle", "values": [0.052, 0.038, 0.061, 0.029, 0.073] } |
| } |
| }, |
| "outputs": { |
| "qT": { "dtype": "float32", "shape": [2, 8], "tolerance": 0.0001, "relTolerance": 0.0001 }, |
| "kT": { "dtype": "float32", "shape": [2, 4], "tolerance": 0.0001, "relTolerance": 0.0001 }, |
| "vT": { "dtype": "float32", "shape": [2, 4], "tolerance": 0.0001, "relTolerance": 0.0001 }, |
| "residualT": { "dtype": "float32", "shape": [2, 64], "tolerance": 0.000001, "relTolerance": 0.000001 } |
| } |
| }, |
| { |
| "name": "f16_skipsum", |
| "attrs": { "K": 32, "Nq": 8, "Nkv": 4, "block_size": 32 }, |
| "inputs": { |
| "aT": { |
| "dtype": "float16", |
| "shape": [2, 32], |
| "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.29, "scale": 0.8 } |
| }, |
| "skipT": { |
| "dtype": "float16", |
| "shape": [2, 32], |
| "data": { "kind": "fillFloat32", "sinStep": 0.37, "cosStep": 0.11, "scale": 0.5 } |
| }, |
| "normScaleT": { |
| "dtype": "float16", |
| "shape": [32], |
| "data": { "kind": "fillFloat32", "sinStep": 0.23, "cosStep": 0.41, "scale": 0.4, "offset": 1.0 } |
| }, |
| "qBT": { |
| "dtype": "uint8", |
| "shape": [8, 1, 16], |
| "data": { "kind": "cycle", "values": [27, 180, 75, 226, 33, 150, 201, 108, 57, 246, 129, 66, 195] } |
| }, |
| "qScalesT": { |
| "dtype": "float16", |
| "shape": [8, 1], |
| "data": { "kind": "cycle", "values": [0.04, 0.055, 0.05, 0.065, 0.06, 0.075, 0.07] } |
| }, |
| "kBT": { |
| "dtype": "uint8", |
| "shape": [4, 1, 16], |
| "data": { "kind": "cycle", "values": [211, 44, 137, 98, 165, 20, 233, 121, 78, 190, 15, 252, 87, 143, 61] } |
| }, |
| "kScalesT": { |
| "dtype": "float16", |
| "shape": [4, 1], |
| "data": { "kind": "cycle", "values": [0.045, 0.03, 0.07, 0.05, 0.08, 0.035] } |
| }, |
| "vBT": { |
| "dtype": "uint8", |
| "shape": [4, 1, 16], |
| "data": { "kind": "cycle", "values": { "$ref": "#/fixtureArrays/norm_gqa_input_vBT" } } |
| }, |
| "vScalesT": { |
| "dtype": "float16", |
| "shape": [4, 1], |
| "data": { "kind": "cycle", "values": [0.052, 0.038, 0.061, 0.029, 0.073] } |
| } |
| }, |
| "outputs": { |
| "qT": { "dtype": "float16", "shape": [2, 8], "tolerance": 0.002, "relTolerance": 0.01 }, |
| "kT": { "dtype": "float16", "shape": [2, 4], "tolerance": 0.002, "relTolerance": 0.01 }, |
| "vT": { "dtype": "float16", "shape": [2, 4], "tolerance": 0.002, "relTolerance": 0.01 }, |
| "residualT": { "dtype": "float16", "shape": [2, 32], "tolerance": 0.002, "relTolerance": 0.002 } |
| } |
| }, |
| { |
| "name": "f16_norm", |
| "attrs": { "K": 32, "Nq": 8, "Nkv": 4, "block_size": 32 }, |
| "inputs": { |
| "aT": { |
| "dtype": "float16", |
| "shape": [3, 32], |
| "data": { "kind": "fillFloat32", "sinStep": 0.13, "cosStep": 0.29, "scale": 0.8 } |
| }, |
| "normScaleT": { |
| "dtype": "float16", |
| "shape": [32], |
| "data": { "kind": "fillFloat32", "sinStep": 0.23, "cosStep": 0.41, "scale": 0.4, "offset": 1.0 } |
| }, |
| "qBT": { |
| "dtype": "uint8", |
| "shape": [8, 1, 16], |
| "data": { "kind": "cycle", "values": [27, 180, 75, 226, 33, 150, 201, 108, 57, 246, 129, 66, 195] } |
| }, |
| "qScalesT": { |
| "dtype": "float16", |
| "shape": [8, 1], |
| "data": { "kind": "cycle", "values": [0.04, 0.055, 0.05, 0.065, 0.06, 0.075, 0.07] } |
| }, |
| "kBT": { |
| "dtype": "uint8", |
| "shape": [4, 1, 16], |
| "data": { "kind": "cycle", "values": [211, 44, 137, 98, 165, 20, 233, 121, 78, 190, 15, 252, 87, 143, 61] } |
| }, |
| "kScalesT": { |
| "dtype": "float16", |
| "shape": [4, 1], |
| "data": { "kind": "cycle", "values": [0.045, 0.03, 0.07, 0.05, 0.08, 0.035] } |
| }, |
| "vBT": { |
| "dtype": "uint8", |
| "shape": [4, 1, 16], |
| "data": { "kind": "cycle", "values": { "$ref": "#/fixtureArrays/norm_gqa_input_vBT" } } |
| }, |
| "vScalesT": { |
| "dtype": "float16", |
| "shape": [4, 1], |
| "data": { "kind": "cycle", "values": [0.052, 0.038, 0.061, 0.029, 0.073] } |
| } |
| }, |
| "outputs": { |
| "qT": { "dtype": "float16", "shape": [3, 8], "tolerance": 0.002, "relTolerance": 0.01 }, |
| "kT": { "dtype": "float16", "shape": [3, 4], "tolerance": 0.002, "relTolerance": 0.01 }, |
| "vT": { "dtype": "float16", "shape": [3, 4], "tolerance": 0.002, "relTolerance": 0.01 } |
| } |
| } |
| ] |
| } |
|
|
