{
  "op": "com.microsoft.MatMulNBitsQkv",
  "tunableSpace": { "TILE_N": [4, 8, 16], "LANES": [4, 8, 16] },
  "cases": [
    {
      "name": "qkv-q4-decode-k2048-nq2048-nkv512",
      "preset": "smoke",
      "vars": { "dtype": "float32" },
      "attrs": { "K": 2048, "Nq": 2048, "Nkv": 512, "block_size": 32 },
      "inputs": {
        "aT": { "shape": [1, 2048], "dtype": "float32", "dist": "normal", "seed": 9101, "scale": 1 },
        "normScaleT": { "shape": [2048], "dtype": "float32", "dist": "normal", "seed": 9102, "scale": 1 },
        "qBT": { "shape": [2048, 64, 16], "dtype": "uint8", "dist": "uniform", "seed": 9103, "scale": 255 },
        "qScalesT": { "shape": [2048, 64], "dtype": "float32", "dist": "normal", "seed": 9104, "scale": 0.05 },
        "kBT": { "shape": [512, 64, 16], "dtype": "uint8", "dist": "uniform", "seed": 9105, "scale": 255 },
        "kScalesT": { "shape": [512, 64], "dtype": "float32", "dist": "normal", "seed": 9106, "scale": 0.05 },
        "vBT": { "shape": [512, 64, 16], "dtype": "uint8", "dist": "uniform", "seed": 9107, "scale": 255 },
        "vScalesT": { "shape": [512, 64], "dtype": "float32", "dist": "normal", "seed": 9108, "scale": 0.05 }
      },
      "outputs": {
        "qT": { "shape": [1, 2048], "dtype": "float32" },
        "kT": { "shape": [1, 512], "dtype": "float32" },
        "vT": { "shape": [1, 512], "dtype": "float32" }
      },
      "bench": { "metrics": [{ "type": "bandwidth", "value": "(2048 + 512 + 512) * 64 * 16 * 4" }] }
    },
    {
      "name": "qkv-q4-llama-decode-k4096-nq4096-nkv4096",
      "preset": "model",
      "provenance": {
        "notes": "Llama class defaults (hidden_size 4096, num_attention_heads 32, no GQA so num_key_value_heads matches) at a decode step. Bytes are GPU storage, not the packed blob: this backend gives a uint8 tensor one byte per u32 element, so a 4-bit code costs 2 bytes of traffic and the projection moves four times its on-disk size. Counting the blob understated the traffic 4x and reported this kernel at a tenth of the bandwidth rail when it runs at about half."
      },
      "vars": { "dtype": "float32", "tokens": 1, "K": 4096, "Nq": 4096, "Nkv": 4096 },
      "attrs": { "K": 4096, "Nq": 4096, "Nkv": 4096, "block_size": 32 },
      "inputs": {
        "aT": { "shape": [1, 4096], "dtype": "float32", "dist": "normal", "seed": 7100, "scale": 0.5 },
        "normScaleT": {
          "shape": [4096],
          "dtype": "float32",
          "dist": "uniform",
          "seed": 7101,
          "scale": 0.1,
          "offset": 1
        },
        "qBT": { "shape": [4096, 128, 16], "dtype": "uint8", "dist": "uniform", "seed": 7102, "min": 0, "max": 255 },
        "qScalesT": {
          "shape": [4096, 128],
          "dtype": "float32",
          "dist": "uniform",
          "seed": 7103,
          "min": 0.005,
          "max": 0.05
        },
        "kBT": { "shape": [4096, 128, 16], "dtype": "uint8", "dist": "uniform", "seed": 7104, "min": 0, "max": 255 },
        "kScalesT": {
          "shape": [4096, 128],
          "dtype": "float32",
          "dist": "uniform",
          "seed": 7105,
          "min": 0.005,
          "max": 0.05
        },
        "vBT": { "shape": [4096, 128, 16], "dtype": "uint8", "dist": "uniform", "seed": 7106, "min": 0, "max": 255 },
        "vScalesT": {
          "shape": [4096, 128],
          "dtype": "float32",
          "dist": "uniform",
          "seed": 7107,
          "min": 0.005,
          "max": 0.05
        }
      },
      "outputs": {
        "qT": { "shape": [1, 4096], "dtype": "float32" },
        "kT": { "shape": [1, 4096], "dtype": "float32" },
        "vT": { "shape": [1, 4096], "dtype": "float32" }
      },
      "bench": { "metrics": [{ "type": "bandwidth", "value": "(args.Nq + 2 * args.Nkv) * args.K * 2" }] }
    },
    {
      "name": "qkv-q4-mistral-decode-k4096-nq4096-nkv1024",
      "preset": "model",
      "provenance": {
        "notes": "Mistral class defaults (hidden_size 4096, 32 query heads, 8 key/value heads, head_dim 128) -- the GQA case, where K and V projections are a quarter of Q. Bytes are GPU storage, not the packed blob: this backend gives a uint8 tensor one byte per u32 element, so a 4-bit code costs 2 bytes of traffic and the projection moves four times its on-disk size. Counting the blob understated the traffic 4x and reported this kernel at a tenth of the bandwidth rail when it runs at about half."
      },
      "vars": { "dtype": "float32", "tokens": 1, "K": 4096, "Nq": 4096, "Nkv": 1024 },
      "attrs": { "K": 4096, "Nq": 4096, "Nkv": 1024, "block_size": 32 },
      "inputs": {
        "aT": { "shape": [1, 4096], "dtype": "float32", "dist": "normal", "seed": 7200, "scale": 0.5 },
        "normScaleT": {
          "shape": [4096],
          "dtype": "float32",
          "dist": "uniform",
          "seed": 7201,
          "scale": 0.1,
          "offset": 1
        },
        "qBT": { "shape": [4096, 128, 16], "dtype": "uint8", "dist": "uniform", "seed": 7202, "min": 0, "max": 255 },
        "qScalesT": {
          "shape": [4096, 128],
          "dtype": "float32",
          "dist": "uniform",
          "seed": 7203,
          "min": 0.005,
          "max": 0.05
        },
        "kBT": { "shape": [1024, 128, 16], "dtype": "uint8", "dist": "uniform", "seed": 7204, "min": 0, "max": 255 },
        "kScalesT": {
          "shape": [1024, 128],
          "dtype": "float32",
          "dist": "uniform",
          "seed": 7205,
          "min": 0.005,
          "max": 0.05
        },
        "vBT": { "shape": [1024, 128, 16], "dtype": "uint8", "dist": "uniform", "seed": 7206, "min": 0, "max": 255 },
        "vScalesT": {
          "shape": [1024, 128],
          "dtype": "float32",
          "dist": "uniform",
          "seed": 7207,
          "min": 0.005,
          "max": 0.05
        }
      },
      "outputs": {
        "qT": { "shape": [1, 4096], "dtype": "float32" },
        "kT": { "shape": [1, 1024], "dtype": "float32" },
        "vT": { "shape": [1, 1024], "dtype": "float32" }
      },
      "bench": { "metrics": [{ "type": "bandwidth", "value": "(args.Nq + 2 * args.Nkv) * args.K * 2" }] }
    },
    {
      "name": "qkv-q4-gemma3-decode-k2304-nq2048-nkv1024",
      "preset": "model",
      "provenance": {
        "notes": "Gemma3 class defaults (hidden_size 2304, 8 query heads, 4 key/value heads, head_dim 256); the projection is wider than hidden, so K does not equal Nq. Bytes are GPU storage, not the packed blob: this backend gives a uint8 tensor one byte per u32 element, so a 4-bit code costs 2 bytes of traffic and the projection moves four times its on-disk size. Counting the blob understated the traffic 4x and reported this kernel at a tenth of the bandwidth rail when it runs at about half."
      },
      "vars": { "dtype": "float32", "tokens": 1, "K": 2304, "Nq": 2048, "Nkv": 1024 },
      "attrs": { "K": 2304, "Nq": 2048, "Nkv": 1024, "block_size": 32 },
      "inputs": {
        "aT": { "shape": [1, 2304], "dtype": "float32", "dist": "normal", "seed": 7300, "scale": 0.5 },
        "normScaleT": {
          "shape": [2304],
          "dtype": "float32",
          "dist": "uniform",
          "seed": 7301,
          "scale": 0.1,
          "offset": 1
        },
        "qBT": { "shape": [2048, 72, 16], "dtype": "uint8", "dist": "uniform", "seed": 7302, "min": 0, "max": 255 },
        "qScalesT": {
          "shape": [2048, 72],
          "dtype": "float32",
          "dist": "uniform",
          "seed": 7303,
          "min": 0.005,
          "max": 0.05
        },
        "kBT": { "shape": [1024, 72, 16], "dtype": "uint8", "dist": "uniform", "seed": 7304, "min": 0, "max": 255 },
        "kScalesT": {
          "shape": [1024, 72],
          "dtype": "float32",
          "dist": "uniform",
          "seed": 7305,
          "min": 0.005,
          "max": 0.05
        },
        "vBT": { "shape": [1024, 72, 16], "dtype": "uint8", "dist": "uniform", "seed": 7306, "min": 0, "max": 255 },
        "vScalesT": {
          "shape": [1024, 72],
          "dtype": "float32",
          "dist": "uniform",
          "seed": 7307,
          "min": 0.005,
          "max": 0.05
        }
      },
      "outputs": {
        "qT": { "shape": [1, 2048], "dtype": "float32" },
        "kT": { "shape": [1, 1024], "dtype": "float32" },
        "vT": { "shape": [1, 1024], "dtype": "float32" }
      },
      "bench": { "metrics": [{ "type": "bandwidth", "value": "(args.Nq + 2 * args.Nkv) * args.K * 2" }] }
    },
    {
      "name": "qkv-q4-phi3-decode-k3072-nq3072-nkv3072",
      "preset": "model",
      "provenance": {
        "notes": "Phi-3 class defaults (hidden_size 3072, num_attention_heads 32, no GQA). Bytes are GPU storage, not the packed blob: this backend gives a uint8 tensor one byte per u32 element, so a 4-bit code costs 2 bytes of traffic and the projection moves four times its on-disk size. Counting the blob understated the traffic 4x and reported this kernel at a tenth of the bandwidth rail when it runs at about half."
      },
      "vars": { "dtype": "float32", "tokens": 1, "K": 3072, "Nq": 3072, "Nkv": 3072 },
      "attrs": { "K": 3072, "Nq": 3072, "Nkv": 3072, "block_size": 32 },
      "inputs": {
        "aT": { "shape": [1, 3072], "dtype": "float32", "dist": "normal", "seed": 7400, "scale": 0.5 },
        "normScaleT": {
          "shape": [3072],
          "dtype": "float32",
          "dist": "uniform",
          "seed": 7401,
          "scale": 0.1,
          "offset": 1
        },
        "qBT": { "shape": [3072, 96, 16], "dtype": "uint8", "dist": "uniform", "seed": 7402, "min": 0, "max": 255 },
        "qScalesT": {
          "shape": [3072, 96],
          "dtype": "float32",
          "dist": "uniform",
          "seed": 7403,
          "min": 0.005,
          "max": 0.05
        },
        "kBT": { "shape": [3072, 96, 16], "dtype": "uint8", "dist": "uniform", "seed": 7404, "min": 0, "max": 255 },
        "kScalesT": {
          "shape": [3072, 96],
          "dtype": "float32",
          "dist": "uniform",
          "seed": 7405,
          "min": 0.005,
          "max": 0.05
        },
        "vBT": { "shape": [3072, 96, 16], "dtype": "uint8", "dist": "uniform", "seed": 7406, "min": 0, "max": 255 },
        "vScalesT": {
          "shape": [3072, 96],
          "dtype": "float32",
          "dist": "uniform",
          "seed": 7407,
          "min": 0.005,
          "max": 0.05
        }
      },
      "outputs": {
        "qT": { "shape": [1, 3072], "dtype": "float32" },
        "kT": { "shape": [1, 3072], "dtype": "float32" },
        "vT": { "shape": [1, 3072], "dtype": "float32" }
      },
      "bench": { "metrics": [{ "type": "bandwidth", "value": "(args.Nq + 2 * args.Nkv) * args.K * 2" }] }
    },
    {
      "name": "qkv-q4-llama-spec8-k4096-nq4096-nkv4096",
      "preset": "model",
      "provenance": {
        "notes": "Llama class defaults with 8 rows, the shape speculative decoding verifies in one pass. Bytes are GPU storage, not the packed blob: this backend gives a uint8 tensor one byte per u32 element, so a 4-bit code costs 2 bytes of traffic and the projection moves four times its on-disk size. Counting the blob understated the traffic 4x and reported this kernel at a tenth of the bandwidth rail when it runs at about half."
      },
      "vars": { "dtype": "float32", "tokens": 8, "K": 4096, "Nq": 4096, "Nkv": 4096 },
      "attrs": { "K": 4096, "Nq": 4096, "Nkv": 4096, "block_size": 32 },
      "inputs": {
        "aT": { "shape": [8, 4096], "dtype": "float32", "dist": "normal", "seed": 7500, "scale": 0.5 },
        "normScaleT": {
          "shape": [4096],
          "dtype": "float32",
          "dist": "uniform",
          "seed": 7501,
          "scale": 0.1,
          "offset": 1
        },
        "qBT": { "shape": [4096, 128, 16], "dtype": "uint8", "dist": "uniform", "seed": 7502, "min": 0, "max": 255 },
        "qScalesT": {
          "shape": [4096, 128],
          "dtype": "float32",
          "dist": "uniform",
          "seed": 7503,
          "min": 0.005,
          "max": 0.05
        },
        "kBT": { "shape": [4096, 128, 16], "dtype": "uint8", "dist": "uniform", "seed": 7504, "min": 0, "max": 255 },
        "kScalesT": {
          "shape": [4096, 128],
          "dtype": "float32",
          "dist": "uniform",
          "seed": 7505,
          "min": 0.005,
          "max": 0.05
        },
        "vBT": { "shape": [4096, 128, 16], "dtype": "uint8", "dist": "uniform", "seed": 7506, "min": 0, "max": 255 },
        "vScalesT": {
          "shape": [4096, 128],
          "dtype": "float32",
          "dist": "uniform",
          "seed": 7507,
          "min": 0.005,
          "max": 0.05
        }
      },
      "outputs": {
        "qT": { "shape": [8, 4096], "dtype": "float32" },
        "kT": { "shape": [8, 4096], "dtype": "float32" },
        "vT": { "shape": [8, 4096], "dtype": "float32" }
      },
      "bench": { "metrics": [{ "type": "bandwidth", "value": "(args.Nq + 2 * args.Nkv) * args.K * 2" }] }
    }
  ]
}