{ "tunableSpace": { "TILE_V": [1, 2, 4], "V_GROUPS": [2, 4, 8], "LANE_WORDS": [2, 4, 8] }, "cases": [ { "name": "gdn-qwen35-27b-decode-f16-t1", "preset": "smoke", "provenance": { "source": "Qwen3.5-27B-A3B gated DeltaNet layer", "notes": "num_heads_q = num_heads_k = 16, num_heads_v = 48, head_size_qk = head_size_v = 128." }, "vars": { "dtype": "float16", "tokens": 1, "batch": 1, "hq": 16, "hv": 48, "dk": 128, "dv": 128, "gates": 2 }, "attrs": { "update_rule": "gated_delta", "gate_activation": "qwen", "beta_activation": "sigmoid", "qk_l2_norm": 1 }, "inputs": { "queryT": { "shape": [1, 16, 128], "dtype": "float16", "dist": "normal", "seed": 401, "scale": 0.2 }, "keyT": { "shape": [1, 16, 128], "dtype": "float16", "dist": "normal", "seed": 402, "scale": 0.2 }, "valueT": { "shape": [1, 48, 128], "dtype": "float16", "dist": "normal", "seed": 403, "scale": 0.2 }, "decayT": { "shape": [1, 48], "dtype": "float32", "dist": "normal", "seed": 404, "scale": 0.1, "offset": 0 }, "betaT": { "shape": [1, 48], "dtype": "float32", "dist": "normal", "seed": 405, "scale": 0.5, "offset": 0 }, "initialStateT": { "shape": [1, 48, 128, 128], "dtype": "float32", "dist": "normal", "seed": 406, "scale": 0.05 }, "aLogT": { "shape": [48], "dtype": "float32", "dist": "normal", "seed": 407, "scale": 0.1 }, "dtBiasT": { "shape": [48], "dtype": "float32", "dist": "normal", "seed": 408, "scale": 0.1 } }, "outputs": { "outputT": { "shape": [1, 48, 128], "dtype": "float16" }, "finalStateT": { "shape": [1, 48, 128, 128], "dtype": "float32" } }, "bench": { "primary": true, "metrics": [ { "type": "bandwidth", "value": "args.tokens * (2 * args.hq * args.dk + 2 * args.hv * args.dv) * dtypeBytes(args.dtype) + args.tokens * args.hv * args.gates * 4 + 2 * args.batch * args.hv * args.dv * args.dk * 4" } ] } }, { "name": "gdn-qwen35-27b-verify-f16-t8", "preset": "smoke", "provenance": { "source": "Qwen3.5-27B-A3B gated DeltaNet layer", "notes": "num_heads_q = num_heads_k = 16, num_heads_v = 48, head_size_qk = head_size_v = 128." }, "vars": { "dtype": "float16", "tokens": 8, "batch": 1, "hq": 16, "hv": 48, "dk": 128, "dv": 128, "gates": 2 }, "attrs": { "update_rule": "gated_delta", "gate_activation": "qwen", "beta_activation": "sigmoid", "qk_l2_norm": 1 }, "inputs": { "queryT": { "shape": [8, 16, 128], "dtype": "float16", "dist": "normal", "seed": 409, "scale": 0.2 }, "keyT": { "shape": [8, 16, 128], "dtype": "float16", "dist": "normal", "seed": 410, "scale": 0.2 }, "valueT": { "shape": [8, 48, 128], "dtype": "float16", "dist": "normal", "seed": 411, "scale": 0.2 }, "decayT": { "shape": [8, 48], "dtype": "float32", "dist": "normal", "seed": 412, "scale": 0.1, "offset": 0 }, "betaT": { "shape": [8, 48], "dtype": "float32", "dist": "normal", "seed": 413, "scale": 0.5, "offset": 0 }, "initialStateT": { "shape": [1, 48, 128, 128], "dtype": "float32", "dist": "normal", "seed": 414, "scale": 0.05 }, "aLogT": { "shape": [48], "dtype": "float32", "dist": "normal", "seed": 415, "scale": 0.1 }, "dtBiasT": { "shape": [48], "dtype": "float32", "dist": "normal", "seed": 416, "scale": 0.1 } }, "outputs": { "outputT": { "shape": [8, 48, 128], "dtype": "float16" }, "finalStateT": { "shape": [1, 48, 128, 128], "dtype": "float32" } }, "bench": { "metrics": [ { "type": "bandwidth", "value": "args.tokens * (2 * args.hq * args.dk + 2 * args.hv * args.dv) * dtypeBytes(args.dtype) + args.tokens * args.hv * args.gates * 4 + 2 * args.batch * args.hv * args.dv * args.dk * 4" } ] } }, { "name": "gdn-qwen35-27b-decode-f32-t1", "preset": "smoke", "provenance": { "source": "Qwen3.5-27B-A3B gated DeltaNet layer", "notes": "num_heads_q = num_heads_k = 16, num_heads_v = 48, head_size_qk = head_size_v = 128." }, "vars": { "dtype": "float32", "tokens": 1, "batch": 1, "hq": 16, "hv": 48, "dk": 128, "dv": 128, "gates": 2 }, "attrs": { "update_rule": "gated_delta", "gate_activation": "qwen", "beta_activation": "sigmoid", "qk_l2_norm": 1 }, "inputs": { "queryT": { "shape": [1, 16, 128], "dtype": "float32", "dist": "normal", "seed": 417, "scale": 0.2 }, "keyT": { "shape": [1, 16, 128], "dtype": "float32", "dist": "normal", "seed": 418, "scale": 0.2 }, "valueT": { "shape": [1, 48, 128], "dtype": "float32", "dist": "normal", "seed": 419, "scale": 0.2 }, "decayT": { "shape": [1, 48], "dtype": "float32", "dist": "normal", "seed": 420, "scale": 0.1, "offset": 0 }, "betaT": { "shape": [1, 48], "dtype": "float32", "dist": "normal", "seed": 421, "scale": 0.5, "offset": 0 }, "initialStateT": { "shape": [1, 48, 128, 128], "dtype": "float32", "dist": "normal", "seed": 422, "scale": 0.05 }, "aLogT": { "shape": [48], "dtype": "float32", "dist": "normal", "seed": 423, "scale": 0.1 }, "dtBiasT": { "shape": [48], "dtype": "float32", "dist": "normal", "seed": 424, "scale": 0.1 } }, "outputs": { "outputT": { "shape": [1, 48, 128], "dtype": "float32" }, "finalStateT": { "shape": [1, 48, 128, 128], "dtype": "float32" } }, "bench": { "metrics": [ { "type": "bandwidth", "value": "args.tokens * (2 * args.hq * args.dk + 2 * args.hv * args.dv) * dtypeBytes(args.dtype) + args.tokens * args.hv * args.gates * 4 + 2 * args.batch * args.hv * args.dv * args.dk * 4" } ] } }, { "name": "gdn-qwen35-27b-verify-batch4-f16-t16", "preset": "smoke", "provenance": { "source": "Qwen3.5-27B-A3B gated DeltaNet layer", "notes": "num_heads_q = num_heads_k = 16, num_heads_v = 48, head_size_qk = head_size_v = 128." }, "vars": { "dtype": "float16", "tokens": 16, "batch": 4, "hq": 16, "hv": 48, "dk": 128, "dv": 128, "gates": 2 }, "attrs": { "update_rule": "gated_delta", "gate_activation": "qwen", "beta_activation": "sigmoid", "qk_l2_norm": 1 }, "inputs": { "queryT": { "shape": [16, 16, 128], "dtype": "float16", "dist": "normal", "seed": 425, "scale": 0.2 }, "keyT": { "shape": [16, 16, 128], "dtype": "float16", "dist": "normal", "seed": 426, "scale": 0.2 }, "valueT": { "shape": [16, 48, 128], "dtype": "float16", "dist": "normal", "seed": 427, "scale": 0.2 }, "decayT": { "shape": [16, 48], "dtype": "float32", "dist": "normal", "seed": 428, "scale": 0.1, "offset": 0 }, "betaT": { "shape": [16, 48], "dtype": "float32", "dist": "normal", "seed": 429, "scale": 0.5, "offset": 0 }, "initialStateT": { "shape": [4, 48, 128, 128], "dtype": "float32", "dist": "normal", "seed": 430, "scale": 0.05 }, "aLogT": { "shape": [48], "dtype": "float32", "dist": "normal", "seed": 431, "scale": 0.1 }, "dtBiasT": { "shape": [48], "dtype": "float32", "dist": "normal", "seed": 432, "scale": 0.1 } }, "outputs": { "outputT": { "shape": [16, 48, 128], "dtype": "float16" }, "finalStateT": { "shape": [4, 48, 128, 128], "dtype": "float32" } }, "bench": { "metrics": [ { "type": "bandwidth", "value": "args.tokens * (2 * args.hq * args.dk + 2 * args.hv * args.dv) * dtypeBytes(args.dtype) + args.tokens * args.hv * args.gates * 4 + 2 * args.batch * args.hv * args.dv * args.dk * 4" } ] } }, { "name": "gdn-qwen3next-decode-f32-t1", "preset": "smoke", "provenance": { "source": "Qwen3-Next gated DeltaNet layer", "notes": "num_heads_q = num_heads_k = 16, num_heads_v = 32, head_size_qk = head_size_v = 128." }, "vars": { "dtype": "float32", "tokens": 1, "batch": 1, "hq": 16, "hv": 32, "dk": 128, "dv": 128, "gates": 2 }, "attrs": { "update_rule": "gated_delta", "gate_activation": "qwen", "beta_activation": "sigmoid", "qk_l2_norm": 1 }, "inputs": { "queryT": { "shape": [1, 16, 128], "dtype": "float32", "dist": "normal", "seed": 433, "scale": 0.2 }, "keyT": { "shape": [1, 16, 128], "dtype": "float32", "dist": "normal", "seed": 434, "scale": 0.2 }, "valueT": { "shape": [1, 32, 128], "dtype": "float32", "dist": "normal", "seed": 435, "scale": 0.2 }, "decayT": { "shape": [1, 32], "dtype": "float32", "dist": "normal", "seed": 436, "scale": 0.1, "offset": 0 }, "betaT": { "shape": [1, 32], "dtype": "float32", "dist": "normal", "seed": 437, "scale": 0.5, "offset": 0 }, "initialStateT": { "shape": [1, 32, 128, 128], "dtype": "float32", "dist": "normal", "seed": 438, "scale": 0.05 }, "aLogT": { "shape": [32], "dtype": "float32", "dist": "normal", "seed": 439, "scale": 0.1 }, "dtBiasT": { "shape": [32], "dtype": "float32", "dist": "normal", "seed": 440, "scale": 0.1 } }, "outputs": { "outputT": { "shape": [1, 32, 128], "dtype": "float32" }, "finalStateT": { "shape": [1, 32, 128, 128], "dtype": "float32" } }, "bench": { "metrics": [ { "type": "bandwidth", "value": "args.tokens * (2 * args.hq * args.dk + 2 * args.hv * args.dv) * dtypeBytes(args.dtype) + args.tokens * args.hv * args.gates * 4 + 2 * args.batch * args.hv * args.dv * args.dk * 4" } ] } }, { "name": "gdn-state16-decode-f32-t1", "preset": "smoke", "provenance": { "source": "state-size floor control", "notes": "Sixteen state heads at 128x128, the state footprint com.microsoft.LinearAttention's qwen3next decode case carries, so the two decode routes move the same recurrent state." }, "vars": { "dtype": "float32", "tokens": 1, "batch": 1, "hq": 16, "hv": 16, "dk": 128, "dv": 128, "gates": 2 }, "attrs": { "update_rule": "gated_delta", "beta_activation": "none" }, "inputs": { "queryT": { "shape": [1, 16, 128], "dtype": "float32", "dist": "normal", "seed": 441, "scale": 0.2 }, "keyT": { "shape": [1, 16, 128], "dtype": "float32", "dist": "normal", "seed": 442, "scale": 0.2 }, "valueT": { "shape": [1, 16, 128], "dtype": "float32", "dist": "normal", "seed": 443, "scale": 0.2 }, "decayT": { "shape": [1, 16], "dtype": "float32", "dist": "normal", "seed": 444, "scale": 0.1, "offset": -0.5 }, "betaT": { "shape": [1, 16], "dtype": "float32", "dist": "normal", "seed": 445, "scale": 0.1, "offset": 0.5 }, "initialStateT": { "shape": [1, 16, 128, 128], "dtype": "float32", "dist": "normal", "seed": 446, "scale": 0.05 } }, "outputs": { "outputT": { "shape": [1, 16, 128], "dtype": "float32" }, "finalStateT": { "shape": [1, 16, 128, 128], "dtype": "float32" } }, "bench": { "metrics": [ { "type": "bandwidth", "value": "args.tokens * (2 * args.hq * args.dk + 2 * args.hv * args.dv) * dtypeBytes(args.dtype) + args.tokens * args.hv * args.gates * 4 + 2 * args.batch * args.hv * args.dv * args.dk * 4" } ] } }, { "name": "gdn-qwen35-27b-prefill-f16-t512", "preset": "smoke", "provenance": { "source": "Qwen3.5-27B-A3B gated DeltaNet layer", "notes": "num_heads_q = num_heads_k = 16, num_heads_v = 48, head_size_qk = head_size_v = 128." }, "vars": { "dtype": "float16", "tokens": 512, "batch": 1, "hq": 16, "hv": 48, "dk": 128, "dv": 128, "gates": 2 }, "attrs": { "update_rule": "gated_delta", "gate_activation": "qwen", "beta_activation": "sigmoid", "qk_l2_norm": 1 }, "inputs": { "queryT": { "shape": [512, 16, 128], "dtype": "float16", "dist": "normal", "seed": 447, "scale": 0.2 }, "keyT": { "shape": [512, 16, 128], "dtype": "float16", "dist": "normal", "seed": 448, "scale": 0.2 }, "valueT": { "shape": [512, 48, 128], "dtype": "float16", "dist": "normal", "seed": 449, "scale": 0.2 }, "decayT": { "shape": [512, 48], "dtype": "float32", "dist": "normal", "seed": 450, "scale": 0.1, "offset": 0 }, "betaT": { "shape": [512, 48], "dtype": "float32", "dist": "normal", "seed": 451, "scale": 0.5, "offset": 0 }, "initialStateT": { "shape": [1, 48, 128, 128], "dtype": "float32", "dist": "normal", "seed": 452, "scale": 0.05 }, "aLogT": { "shape": [48], "dtype": "float32", "dist": "normal", "seed": 453, "scale": 0.1 }, "dtBiasT": { "shape": [48], "dtype": "float32", "dist": "normal", "seed": 454, "scale": 0.1 } }, "outputs": { "outputT": { "shape": [512, 48, 128], "dtype": "float16" }, "finalStateT": { "shape": [1, 48, 128, 128], "dtype": "float32" } }, "bench": { "metrics": [{ "type": "gflops", "value": "2 * 3 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "gdn-nonsquare-prefill-f16-dk64-dv128-t256", "preset": "smoke", "provenance": { "source": "non-square head geometry", "notes": "head_size_qk 64 with head_size_v 128, where the V-major state stride and the key-reduction width differ." }, "vars": { "dtype": "float16", "tokens": 256, "batch": 1, "hq": 16, "hv": 32, "dk": 64, "dv": 128, "gates": 2 }, "attrs": { "update_rule": "gated_delta", "beta_activation": "sigmoid", "qk_l2_norm": 1 }, "inputs": { "queryT": { "shape": [256, 16, 64], "dtype": "float16", "dist": "normal", "seed": 455, "scale": 0.2 }, "keyT": { "shape": [256, 16, 64], "dtype": "float16", "dist": "normal", "seed": 456, "scale": 0.2 }, "valueT": { "shape": [256, 32, 128], "dtype": "float16", "dist": "normal", "seed": 457, "scale": 0.2 }, "decayT": { "shape": [256, 32], "dtype": "float32", "dist": "normal", "seed": 458, "scale": 0.1, "offset": -0.5 }, "betaT": { "shape": [256, 32], "dtype": "float32", "dist": "normal", "seed": 459, "scale": 0.5, "offset": 0 }, "initialStateT": { "shape": [1, 32, 128, 64], "dtype": "float32", "dist": "normal", "seed": 460, "scale": 0.05 } }, "outputs": { "outputT": { "shape": [256, 32, 128], "dtype": "float16" }, "finalStateT": { "shape": [1, 32, 128, 64], "dtype": "float32" } }, "bench": { "metrics": [{ "type": "gflops", "value": "2 * 3 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "gdn-ragged-mixed-f16-t1027", "preset": "smoke", "provenance": { "source": "mixed ragged prefill plus decode in one call", "notes": "One 1024-token prefill request followed by three single-token requests." }, "vars": { "dtype": "float16", "tokens": 1027, "batch": 4, "hq": 16, "hv": 48, "dk": 128, "dv": 128, "gates": 2 }, "attrs": { "update_rule": "gated_delta", "gate_activation": "qwen", "beta_activation": "sigmoid", "qk_l2_norm": 1 }, "inputs": { "queryT": { "shape": [1027, 16, 128], "dtype": "float16", "dist": "normal", "seed": 461, "scale": 0.2 }, "keyT": { "shape": [1027, 16, 128], "dtype": "float16", "dist": "normal", "seed": 462, "scale": 0.2 }, "valueT": { "shape": [1027, 48, 128], "dtype": "float16", "dist": "normal", "seed": 463, "scale": 0.2 }, "cuSeqlensT": { "shape": [5], "dtype": "int32", "data": { "kind": "values", "values": [0, 1024, 1025, 1026, 1027] } }, "decayT": { "shape": [1027, 48], "dtype": "float32", "dist": "normal", "seed": 464, "scale": 0.1, "offset": 0 }, "betaT": { "shape": [1027, 48], "dtype": "float32", "dist": "normal", "seed": 465, "scale": 0.5, "offset": 0 }, "initialStateT": { "shape": [4, 48, 128, 128], "dtype": "float32", "dist": "normal", "seed": 466, "scale": 0.05 }, "aLogT": { "shape": [48], "dtype": "float32", "dist": "normal", "seed": 467, "scale": 0.1 }, "dtBiasT": { "shape": [48], "dtype": "float32", "dist": "normal", "seed": 468, "scale": 0.1 } }, "outputs": { "outputT": { "shape": [1027, 48, 128], "dtype": "float16" }, "finalStateT": { "shape": [4, 48, 128, 128], "dtype": "float32" } }, "bench": { "metrics": [{ "type": "gflops", "value": "2 * 3 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "gdn-chunk-boundary-f16-t65", "preset": "smoke", "provenance": { "source": "chunk-boundary sweep", "notes": "One token past a 64-token chunk." }, "vars": { "dtype": "float16", "tokens": 65, "batch": 1, "hq": 4, "hv": 8, "dk": 128, "dv": 128, "gates": 2 }, "attrs": { "update_rule": "gated_delta", "gate_activation": "qwen", "beta_activation": "sigmoid", "qk_l2_norm": 1 }, "inputs": { "queryT": { "shape": [65, 4, 128], "dtype": "float16", "dist": "normal", "seed": 469, "scale": 0.2 }, "keyT": { "shape": [65, 4, 128], "dtype": "float16", "dist": "normal", "seed": 470, "scale": 0.2 }, "valueT": { "shape": [65, 8, 128], "dtype": "float16", "dist": "normal", "seed": 471, "scale": 0.2 }, "decayT": { "shape": [65, 8], "dtype": "float32", "dist": "normal", "seed": 472, "scale": 0.1, "offset": 0 }, "betaT": { "shape": [65, 8], "dtype": "float32", "dist": "normal", "seed": 473, "scale": 0.5, "offset": 0 }, "initialStateT": { "shape": [1, 8, 128, 128], "dtype": "float32", "dist": "normal", "seed": 474, "scale": 0.05 }, "aLogT": { "shape": [8], "dtype": "float32", "dist": "normal", "seed": 475, "scale": 0.1 }, "dtBiasT": { "shape": [8], "dtype": "float32", "dist": "normal", "seed": 476, "scale": 0.1 } }, "outputs": { "outputT": { "shape": [65, 8, 128], "dtype": "float16" }, "finalStateT": { "shape": [1, 8, 128, 128], "dtype": "float32" } }, "bench": { "metrics": [{ "type": "gflops", "value": "2 * 3 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "gdn-chunk-boundary-f16-t130", "preset": "smoke", "provenance": { "source": "chunk-boundary sweep", "notes": "Two 64-token chunks plus two tokens." }, "vars": { "dtype": "float16", "tokens": 130, "batch": 1, "hq": 4, "hv": 8, "dk": 128, "dv": 128, "gates": 2 }, "attrs": { "update_rule": "gated_delta", "gate_activation": "qwen", "beta_activation": "sigmoid", "qk_l2_norm": 1 }, "inputs": { "queryT": { "shape": [130, 4, 128], "dtype": "float16", "dist": "normal", "seed": 477, "scale": 0.2 }, "keyT": { "shape": [130, 4, 128], "dtype": "float16", "dist": "normal", "seed": 478, "scale": 0.2 }, "valueT": { "shape": [130, 8, 128], "dtype": "float16", "dist": "normal", "seed": 479, "scale": 0.2 }, "decayT": { "shape": [130, 8], "dtype": "float32", "dist": "normal", "seed": 480, "scale": 0.1, "offset": 0 }, "betaT": { "shape": [130, 8], "dtype": "float32", "dist": "normal", "seed": 481, "scale": 0.5, "offset": 0 }, "initialStateT": { "shape": [1, 8, 128, 128], "dtype": "float32", "dist": "normal", "seed": 482, "scale": 0.05 }, "aLogT": { "shape": [8], "dtype": "float32", "dist": "normal", "seed": 483, "scale": 0.1 }, "dtBiasT": { "shape": [8], "dtype": "float32", "dist": "normal", "seed": 484, "scale": 0.1 } }, "outputs": { "outputT": { "shape": [130, 8, 128], "dtype": "float16" }, "finalStateT": { "shape": [1, 8, 128, 128], "dtype": "float32" } }, "bench": { "metrics": [{ "type": "gflops", "value": "2 * 3 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "gdn-linear-rule-prefill-f16-t512", "preset": "smoke", "provenance": { "source": "rule coverage", "notes": "The linear rule reads no gate stream and skips the delta retrieval, so it is the arithmetic floor of the family." }, "vars": { "dtype": "float16", "tokens": 512, "batch": 1, "hq": 16, "hv": 48, "dk": 128, "dv": 128, "gates": 0 }, "attrs": { "update_rule": "linear" }, "inputs": { "queryT": { "shape": [512, 16, 128], "dtype": "float16", "dist": "normal", "seed": 485, "scale": 0.2 }, "keyT": { "shape": [512, 16, 128], "dtype": "float16", "dist": "normal", "seed": 486, "scale": 0.2 }, "valueT": { "shape": [512, 48, 128], "dtype": "float16", "dist": "normal", "seed": 487, "scale": 0.2 }, "initialStateT": { "shape": [1, 48, 128, 128], "dtype": "float32", "dist": "normal", "seed": 488, "scale": 0.05 } }, "outputs": { "outputT": { "shape": [512, 48, 128], "dtype": "float16" }, "finalStateT": { "shape": [1, 48, 128, 128], "dtype": "float32" } }, "bench": { "metrics": [{ "type": "gflops", "value": "2 * 2 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "gdn-qwen35-27b-decode-zero-state-f16-t1", "preset": "smoke", "provenance": { "source": "Qwen3.5-27B-A3B gated DeltaNet layer", "notes": "num_heads_q = num_heads_k = 16, num_heads_v = 48, head_size_qk = head_size_v = 128." }, "vars": { "dtype": "float16", "tokens": 1, "batch": 1, "hq": 16, "hv": 48, "dk": 128, "dv": 128, "gates": 2 }, "attrs": { "update_rule": "gated_delta", "gate_activation": "qwen", "beta_activation": "sigmoid", "qk_l2_norm": 1 }, "inputs": { "queryT": { "shape": [1, 16, 128], "dtype": "float16", "dist": "normal", "seed": 489, "scale": 0.2 }, "keyT": { "shape": [1, 16, 128], "dtype": "float16", "dist": "normal", "seed": 490, "scale": 0.2 }, "valueT": { "shape": [1, 48, 128], "dtype": "float16", "dist": "normal", "seed": 491, "scale": 0.2 }, "cuSeqlensT": { "shape": [2], "dtype": "int32", "data": { "kind": "values", "values": [0, 1] } }, "decayT": { "shape": [1, 48], "dtype": "float32", "dist": "normal", "seed": 492, "scale": 0.1, "offset": 0 }, "betaT": { "shape": [1, 48], "dtype": "float32", "dist": "normal", "seed": 493, "scale": 0.5, "offset": 0 }, "aLogT": { "shape": [48], "dtype": "float32", "dist": "normal", "seed": 494, "scale": 0.1 }, "dtBiasT": { "shape": [48], "dtype": "float32", "dist": "normal", "seed": 495, "scale": 0.1 } }, "outputs": { "outputT": { "shape": [1, 48, 128], "dtype": "float16" }, "finalStateT": { "shape": [1, 48, 128, 128], "dtype": "float32" } }, "bench": { "metrics": [ { "type": "bandwidth", "value": "args.tokens * (2 * args.hq * args.dk + 2 * args.hv * args.dv) * dtypeBytes(args.dtype) + args.tokens * args.hv * args.gates * 4 + 2 * args.batch * args.hv * args.dv * args.dk * 4" } ] } }, { "name": "gdn-qwen35-27b-prefill-f16-t2048", "preset": "scale", "provenance": { "source": "Qwen3.5-27B-A3B gated DeltaNet layer", "notes": "num_heads_q = num_heads_k = 16, num_heads_v = 48, head_size_qk = head_size_v = 128." }, "vars": { "dtype": "float16", "tokens": 2048, "batch": 1, "hq": 16, "hv": 48, "dk": 128, "dv": 128, "gates": 2 }, "attrs": { "update_rule": "gated_delta", "gate_activation": "qwen", "beta_activation": "sigmoid", "qk_l2_norm": 1 }, "inputs": { "queryT": { "shape": [2048, 16, 128], "dtype": "float16", "dist": "normal", "seed": 496, "scale": 0.2 }, "keyT": { "shape": [2048, 16, 128], "dtype": "float16", "dist": "normal", "seed": 497, "scale": 0.2 }, "valueT": { "shape": [2048, 48, 128], "dtype": "float16", "dist": "normal", "seed": 498, "scale": 0.2 }, "decayT": { "shape": [2048, 48], "dtype": "float32", "dist": "normal", "seed": 499, "scale": 0.1, "offset": 0 }, "betaT": { "shape": [2048, 48], "dtype": "float32", "dist": "normal", "seed": 500, "scale": 0.5, "offset": 0 }, "initialStateT": { "shape": [1, 48, 128, 128], "dtype": "float32", "dist": "normal", "seed": 501, "scale": 0.05 }, "aLogT": { "shape": [48], "dtype": "float32", "dist": "normal", "seed": 502, "scale": 0.1 }, "dtBiasT": { "shape": [48], "dtype": "float32", "dist": "normal", "seed": 503, "scale": 0.1 } }, "outputs": { "outputT": { "shape": [2048, 48, 128], "dtype": "float16" }, "finalStateT": { "shape": [1, 48, 128, 128], "dtype": "float32" } }, "bench": { "metrics": [{ "type": "gflops", "value": "2 * 3 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "gdn-qwen35-27b-decode-batch16-f16-t16", "preset": "scale", "provenance": { "source": "Qwen3.5-27B-A3B gated DeltaNet layer", "notes": "num_heads_q = num_heads_k = 16, num_heads_v = 48, head_size_qk = head_size_v = 128." }, "vars": { "dtype": "float16", "tokens": 16, "batch": 16, "hq": 16, "hv": 48, "dk": 128, "dv": 128, "gates": 2 }, "attrs": { "update_rule": "gated_delta", "gate_activation": "qwen", "beta_activation": "sigmoid", "qk_l2_norm": 1 }, "inputs": { "queryT": { "shape": [16, 16, 128], "dtype": "float16", "dist": "normal", "seed": 504, "scale": 0.2 }, "keyT": { "shape": [16, 16, 128], "dtype": "float16", "dist": "normal", "seed": 505, "scale": 0.2 }, "valueT": { "shape": [16, 48, 128], "dtype": "float16", "dist": "normal", "seed": 506, "scale": 0.2 }, "decayT": { "shape": [16, 48], "dtype": "float32", "dist": "normal", "seed": 507, "scale": 0.1, "offset": 0 }, "betaT": { "shape": [16, 48], "dtype": "float32", "dist": "normal", "seed": 508, "scale": 0.5, "offset": 0 }, "initialStateT": { "shape": [16, 48, 128, 128], "dtype": "float32", "dist": "normal", "seed": 509, "scale": 0.05 }, "aLogT": { "shape": [48], "dtype": "float32", "dist": "normal", "seed": 510, "scale": 0.1 }, "dtBiasT": { "shape": [48], "dtype": "float32", "dist": "normal", "seed": 511, "scale": 0.1 } }, "outputs": { "outputT": { "shape": [16, 48, 128], "dtype": "float16" }, "finalStateT": { "shape": [16, 48, 128, 128], "dtype": "float32" } }, "bench": { "metrics": [ { "type": "bandwidth", "value": "args.tokens * (2 * args.hq * args.dk + 2 * args.hv * args.dv) * dtypeBytes(args.dtype) + args.tokens * args.hv * args.gates * 4 + 2 * args.batch * args.hv * args.dv * args.dk * 4" } ] } }, { "name": "gdn-qwen35-27b-chunked-hint-f16-t512", "preset": "scale", "provenance": { "source": "Qwen3.5-27B-A3B gated DeltaNet layer", "notes": "num_heads_q = num_heads_k = 16, num_heads_v = 48, head_size_qk = head_size_v = 128." }, "vars": { "dtype": "float16", "tokens": 512, "batch": 1, "hq": 16, "hv": 48, "dk": 128, "dv": 128, "gates": 2 }, "attrs": { "update_rule": "gated_delta", "gate_activation": "qwen", "beta_activation": "sigmoid", "qk_l2_norm": 1, "chunk_size": 32 }, "inputs": { "queryT": { "shape": [512, 16, 128], "dtype": "float16", "dist": "normal", "seed": 512, "scale": 0.2 }, "keyT": { "shape": [512, 16, 128], "dtype": "float16", "dist": "normal", "seed": 513, "scale": 0.2 }, "valueT": { "shape": [512, 48, 128], "dtype": "float16", "dist": "normal", "seed": 514, "scale": 0.2 }, "decayT": { "shape": [512, 48], "dtype": "float32", "dist": "normal", "seed": 515, "scale": 0.1, "offset": 0 }, "betaT": { "shape": [512, 48], "dtype": "float32", "dist": "normal", "seed": 516, "scale": 0.5, "offset": 0 }, "initialStateT": { "shape": [1, 48, 128, 128], "dtype": "float32", "dist": "normal", "seed": 517, "scale": 0.05 }, "aLogT": { "shape": [48], "dtype": "float32", "dist": "normal", "seed": 518, "scale": 0.1 }, "dtBiasT": { "shape": [48], "dtype": "float32", "dist": "normal", "seed": 519, "scale": 0.1 } }, "outputs": { "outputT": { "shape": [512, 48, 128], "dtype": "float16" }, "finalStateT": { "shape": [1, 48, 128, 128], "dtype": "float32" } }, "bench": { "metrics": [{ "type": "gflops", "value": "2 * 3 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "gdn-general-float32-gated_delta-dk7-dv5-h1x2-b1-t33-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [1, 33, 1, 7], "dtype": "float32", "dist": "normal", "seed": 59300, "scale": 0.2 }, "keyT": { "shape": [1, 33, 1, 7], "dtype": "float32", "dist": "normal", "seed": 59301, "scale": 0.2 }, "valueT": { "shape": [1, 33, 2, 5], "dtype": "float32", "dist": "normal", "seed": 59302, "scale": 0.2 }, "decayT": { "shape": [1, 33, 2], "dtype": "float32", "dist": "normal", "seed": 59303, "scale": 0.1 }, "aLogT": { "shape": [2], "dtype": "float32", "dist": "normal", "seed": 59304, "scale": 0.1 }, "dtBiasT": { "shape": [2], "dtype": "float32", "dist": "normal", "seed": 59305, "scale": 0.1 }, "betaT": { "shape": [1, 33, 2], "dtype": "float32", "dist": "normal", "seed": 59306, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [1, 33, 2, 5] }, "finalStateT": { "dtype": "float32", "shape": [1, 2, 5, 7] } } }, { "name": "gdn-general-float32-delta-dk20-dv6-h1x3-b1-t65-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "delta", "qk_l2_norm": 1, "beta_activation": "sigmoid" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [65, 1, 20], "dtype": "float32", "dist": "normal", "seed": 59311, "scale": 0.2 }, "keyT": { "shape": [65, 1, 20], "dtype": "float32", "dist": "normal", "seed": 59312, "scale": 0.2 }, "valueT": { "shape": [65, 3, 6], "dtype": "float32", "dist": "normal", "seed": 59313, "scale": 0.2 }, "initialStateT": { "shape": [1, 3, 6, 20], "dtype": "float32", "dist": "normal", "seed": 59314, "scale": 0.05 }, "cuSeqlensT": { "shape": [2], "dtype": "int32", "data": { "kind": "values", "values": [0, 65] } }, "betaT": { "shape": [65, 3], "dtype": "float32", "dist": "normal", "seed": 59315, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [65, 3, 6] }, "finalStateT": { "dtype": "float32", "shape": [1, 3, 6, 20] } } }, { "name": "gdn-general-float32-linear-dk32-dv8-h2x6-b2-t17-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "linear", "qk_l2_norm": 1 }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [34, 2, 32], "dtype": "float32", "dist": "normal", "seed": 59322, "scale": 0.2 }, "keyT": { "shape": [34, 2, 32], "dtype": "float32", "dist": "normal", "seed": 59323, "scale": 0.2 }, "valueT": { "shape": [34, 6, 8], "dtype": "float32", "dist": "normal", "seed": 59324, "scale": 0.2 }, "initialStateT": { "shape": [2, 6, 8, 32], "dtype": "float32", "dist": "normal", "seed": 59325, "scale": 0.05 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [34, 6, 8] }, "finalStateT": { "dtype": "float32", "shape": [2, 6, 8, 32] } } }, { "name": "gdn-general-float32-gated-dk64-dv16-h2x4-b1-t129-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "gated", "qk_l2_norm": 1, "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [1, 129, 2, 64], "dtype": "float32", "dist": "normal", "seed": 59333, "scale": 0.2 }, "keyT": { "shape": [1, 129, 2, 64], "dtype": "float32", "dist": "normal", "seed": 59334, "scale": 0.2 }, "valueT": { "shape": [1, 129, 4, 16], "dtype": "float32", "dist": "normal", "seed": 59335, "scale": 0.2 }, "initialStateT": { "shape": [1, 4, 16, 64], "dtype": "float32", "dist": "normal", "seed": 59336, "scale": 0.05 }, "decayT": { "shape": [1, 129, 4], "dtype": "float32", "dist": "normal", "seed": 59337, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59338, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59339, "scale": 0.1 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [1, 129, 4, 16] }, "finalStateT": { "dtype": "float32", "shape": [1, 4, 16, 64] } } }, { "name": "gdn-general-float32-gated_delta-dk96-dv1-h1x1-b1-t33-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [33, 1, 96], "dtype": "float32", "dist": "normal", "seed": 59344, "scale": 0.2 }, "keyT": { "shape": [33, 1, 96], "dtype": "float32", "dist": "normal", "seed": 59345, "scale": 0.2 }, "valueT": { "shape": [33, 1, 1], "dtype": "float32", "dist": "normal", "seed": 59346, "scale": 0.2 }, "cuSeqlensT": { "shape": [2], "dtype": "int32", "data": { "kind": "values", "values": [0, 33] } }, "decayT": { "shape": [33, 1], "dtype": "float32", "dist": "normal", "seed": 59347, "scale": 0.1 }, "aLogT": { "shape": [1], "dtype": "float32", "dist": "normal", "seed": 59348, "scale": 0.1 }, "dtBiasT": { "shape": [1], "dtype": "float32", "dist": "normal", "seed": 59349, "scale": 0.1 }, "betaT": { "shape": [33, 1], "dtype": "float32", "dist": "normal", "seed": 59350, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [33, 1, 1] }, "finalStateT": { "dtype": "float32", "shape": [1, 1, 1, 96] } } }, { "name": "gdn-general-float32-gated_delta-dk128-dv18-h2x6-b1-t65-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [65, 2, 128], "dtype": "float32", "dist": "normal", "seed": 59355, "scale": 0.2 }, "keyT": { "shape": [65, 2, 128], "dtype": "float32", "dist": "normal", "seed": 59356, "scale": 0.2 }, "valueT": { "shape": [65, 6, 18], "dtype": "float32", "dist": "normal", "seed": 59357, "scale": 0.2 }, "initialStateT": { "shape": [1, 6, 18, 128], "dtype": "float32", "dist": "normal", "seed": 59358, "scale": 0.05 }, "cuSeqlensT": { "shape": [2], "dtype": "int32", "data": { "kind": "values", "values": [0, 65] } }, "decayT": { "shape": [65, 6], "dtype": "float32", "dist": "normal", "seed": 59359, "scale": 0.1 }, "aLogT": { "shape": [6], "dtype": "float32", "dist": "normal", "seed": 59360, "scale": 0.1 }, "dtBiasT": { "shape": [6], "dtype": "float32", "dist": "normal", "seed": 59361, "scale": 0.1 }, "betaT": { "shape": [65, 6], "dtype": "float32", "dist": "normal", "seed": 59362, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [65, 6, 18] }, "finalStateT": { "dtype": "float32", "shape": [1, 6, 18, 128] } } }, { "name": "gdn-general-float32-gated_delta-dk128-dv32-h2x6-b1-t257-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [1, 257, 2, 128], "dtype": "float32", "dist": "normal", "seed": 59366, "scale": 0.2 }, "keyT": { "shape": [1, 257, 2, 128], "dtype": "float32", "dist": "normal", "seed": 59367, "scale": 0.2 }, "valueT": { "shape": [1, 257, 6, 32], "dtype": "float32", "dist": "normal", "seed": 59368, "scale": 0.2 }, "initialStateT": { "shape": [1, 6, 32, 128], "dtype": "float32", "dist": "normal", "seed": 59369, "scale": 0.05 }, "decayT": { "shape": [1, 257, 6], "dtype": "float32", "dist": "normal", "seed": 59370, "scale": 0.1 }, "aLogT": { "shape": [6], "dtype": "float32", "dist": "normal", "seed": 59371, "scale": 0.1 }, "dtBiasT": { "shape": [6], "dtype": "float32", "dist": "normal", "seed": 59372, "scale": 0.1 }, "betaT": { "shape": [1, 257, 6], "dtype": "float32", "dist": "normal", "seed": 59373, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [1, 257, 6, 32] }, "finalStateT": { "dtype": "float32", "shape": [1, 6, 32, 128] } } }, { "name": "gdn-general-float32-gated_delta-dk256-dv64-h2x4-b2-t33-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [66, 2, 256], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [66, 2, 256], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [66, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 256], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [66, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [66, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [66, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 256] } } }, { "name": "gdn-general-float32-linear-dk64-dv9-h1x3-b1-t65-tile4", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "linear", "qk_l2_norm": 1 }, "tunables": { "TILE_V": 4 }, "inputs": { "queryT": { "shape": [65, 1, 64], "dtype": "float32", "dist": "normal", "seed": 59388, "scale": 0.2 }, "keyT": { "shape": [65, 1, 64], "dtype": "float32", "dist": "normal", "seed": 59389, "scale": 0.2 }, "valueT": { "shape": [65, 3, 9], "dtype": "float32", "dist": "normal", "seed": 59390, "scale": 0.2 }, "cuSeqlensT": { "shape": [2], "dtype": "int32", "data": { "kind": "values", "values": [0, 65] } } }, "outputs": { "outputT": { "dtype": "float32", "shape": [65, 3, 9] }, "finalStateT": { "dtype": "float32", "shape": [1, 3, 9, 64] } } }, { "name": "gdn-general-float32-gated_delta-dk128-dv20-h2x6-b1-t33-tile4", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 4 }, "inputs": { "queryT": { "shape": [1, 33, 2, 128], "dtype": "float32", "dist": "normal", "seed": 59399, "scale": 0.2 }, "keyT": { "shape": [1, 33, 2, 128], "dtype": "float32", "dist": "normal", "seed": 59400, "scale": 0.2 }, "valueT": { "shape": [1, 33, 6, 20], "dtype": "float32", "dist": "normal", "seed": 59401, "scale": 0.2 }, "initialStateT": { "shape": [1, 6, 20, 128], "dtype": "float32", "dist": "normal", "seed": 59402, "scale": 0.05 }, "decayT": { "shape": [1, 33, 6], "dtype": "float32", "dist": "normal", "seed": 59403, "scale": 0.1 }, "aLogT": { "shape": [6], "dtype": "float32", "dist": "normal", "seed": 59404, "scale": 0.1 }, "dtBiasT": { "shape": [6], "dtype": "float32", "dist": "normal", "seed": 59405, "scale": 0.1 }, "betaT": { "shape": [1, 33, 6], "dtype": "float32", "dist": "normal", "seed": 59406, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [1, 33, 6, 20] }, "finalStateT": { "dtype": "float32", "shape": [1, 6, 20, 128] } } }, { "name": "gdn-general-float32-delta-dk32-dv12-h2x4-b1-t129-tile4", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "delta", "qk_l2_norm": 1, "beta_activation": "sigmoid" }, "tunables": { "TILE_V": 4 }, "inputs": { "queryT": { "shape": [129, 2, 32], "dtype": "float32", "dist": "normal", "seed": 59410, "scale": 0.2 }, "keyT": { "shape": [129, 2, 32], "dtype": "float32", "dist": "normal", "seed": 59411, "scale": 0.2 }, "valueT": { "shape": [129, 4, 12], "dtype": "float32", "dist": "normal", "seed": 59412, "scale": 0.2 }, "initialStateT": { "shape": [1, 4, 12, 32], "dtype": "float32", "dist": "normal", "seed": 59413, "scale": 0.05 }, "betaT": { "shape": [129, 4], "dtype": "float32", "dist": "normal", "seed": 59414, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [129, 4, 12] }, "finalStateT": { "dtype": "float32", "shape": [1, 4, 12, 32] } } }, { "name": "gdn-general-float32-gated-dk8-dv4-h1x2-b1-t65-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "gated", "qk_l2_norm": 1, "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [65, 1, 8], "dtype": "float32", "dist": "normal", "seed": 59421, "scale": 0.2 }, "keyT": { "shape": [65, 1, 8], "dtype": "float32", "dist": "normal", "seed": 59422, "scale": 0.2 }, "valueT": { "shape": [65, 2, 4], "dtype": "float32", "dist": "normal", "seed": 59423, "scale": 0.2 }, "initialStateT": { "shape": [1, 2, 4, 8], "dtype": "float32", "dist": "normal", "seed": 59424, "scale": 0.05 }, "decayT": { "shape": [65, 2], "dtype": "float32", "dist": "normal", "seed": 59425, "scale": 0.1 }, "aLogT": { "shape": [2], "dtype": "float32", "dist": "normal", "seed": 59426, "scale": 0.1 }, "dtBiasT": { "shape": [2], "dtype": "float32", "dist": "normal", "seed": 59427, "scale": 0.1 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [65, 2, 4] }, "finalStateT": { "dtype": "float32", "shape": [1, 2, 4, 8] } } }, { "name": "gdn-general-float32-gated_delta-dk128-dv32-h2x4-b1-t1-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [1, 1, 2, 128], "dtype": "float32", "dist": "normal", "seed": 59432, "scale": 0.2 }, "keyT": { "shape": [1, 1, 2, 128], "dtype": "float32", "dist": "normal", "seed": 59433, "scale": 0.2 }, "valueT": { "shape": [1, 1, 4, 32], "dtype": "float32", "dist": "normal", "seed": 59434, "scale": 0.2 }, "decayT": { "shape": [1, 1, 4], "dtype": "float32", "dist": "normal", "seed": 59435, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59436, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59437, "scale": 0.1 }, "betaT": { "shape": [1, 1, 4], "dtype": "float32", "dist": "normal", "seed": 59438, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [1, 1, 4, 32] }, "finalStateT": { "dtype": "float32", "shape": [1, 4, 32, 128] } } }, { "name": "gdn-general-float32-gated_delta-dk128-dv32-h2x4-b4-t4-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [16, 2, 128], "dtype": "float32", "dist": "normal", "seed": 59443, "scale": 0.2 }, "keyT": { "shape": [16, 2, 128], "dtype": "float32", "dist": "normal", "seed": 59444, "scale": 0.2 }, "valueT": { "shape": [16, 4, 32], "dtype": "float32", "dist": "normal", "seed": 59445, "scale": 0.2 }, "initialStateT": { "shape": [4, 4, 32, 128], "dtype": "float32", "dist": "normal", "seed": 59446, "scale": 0.05 }, "cuSeqlensT": { "shape": [5], "dtype": "int32", "data": { "kind": "values", "values": [0, 4, 8, 12, 16] } }, "decayT": { "shape": [16, 4], "dtype": "float32", "dist": "normal", "seed": 59447, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59448, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59449, "scale": 0.1 }, "betaT": { "shape": [16, 4], "dtype": "float32", "dist": "normal", "seed": 59450, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [16, 4, 32] }, "finalStateT": { "dtype": "float32", "shape": [4, 4, 32, 128] } } }, { "name": "gdn-general-float32-linear-dk64-dv64-h2x4-b1-t65-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "linear", "qk_l2_norm": 1 }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [65, 2, 64], "dtype": "float32", "dist": "normal", "seed": 59454, "scale": 0.2 }, "keyT": { "shape": [65, 2, 64], "dtype": "float32", "dist": "normal", "seed": 59455, "scale": 0.2 }, "valueT": { "shape": [65, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59456, "scale": 0.2 }, "initialStateT": { "shape": [1, 4, 64, 64], "dtype": "float32", "dist": "normal", "seed": 59457, "scale": 0.05 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [65, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [1, 4, 64, 64] } } }, { "name": "gdn-general-float32-gated_delta-dk128-dv64-h2x4-b2-t129-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [2, 129, 2, 128], "dtype": "float32", "dist": "normal", "seed": 59465, "scale": 0.2 }, "keyT": { "shape": [2, 129, 2, 128], "dtype": "float32", "dist": "normal", "seed": 59466, "scale": 0.2 }, "valueT": { "shape": [2, 129, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59467, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 128], "dtype": "float32", "dist": "normal", "seed": 59468, "scale": 0.05 }, "decayT": { "shape": [2, 129, 4], "dtype": "float32", "dist": "normal", "seed": 59469, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59470, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59471, "scale": 0.1 }, "betaT": { "shape": [2, 129, 4], "dtype": "float32", "dist": "normal", "seed": 59472, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [2, 129, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 128] } } }, { "name": "gdn-general-float16-gated_delta-dk7-dv5-h1x2-b1-t33-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [1, 33, 1, 7], "dtype": "float16", "dist": "normal", "seed": 59300, "scale": 0.2 }, "keyT": { "shape": [1, 33, 1, 7], "dtype": "float16", "dist": "normal", "seed": 59301, "scale": 0.2 }, "valueT": { "shape": [1, 33, 2, 5], "dtype": "float16", "dist": "normal", "seed": 59302, "scale": 0.2 }, "decayT": { "shape": [1, 33, 2], "dtype": "float32", "dist": "normal", "seed": 59303, "scale": 0.1 }, "aLogT": { "shape": [2], "dtype": "float32", "dist": "normal", "seed": 59304, "scale": 0.1 }, "dtBiasT": { "shape": [2], "dtype": "float32", "dist": "normal", "seed": 59305, "scale": 0.1 }, "betaT": { "shape": [1, 33, 2], "dtype": "float32", "dist": "normal", "seed": 59306, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [1, 33, 2, 5] }, "finalStateT": { "dtype": "float32", "shape": [1, 2, 5, 7] } } }, { "name": "gdn-general-float16-delta-dk20-dv6-h1x3-b1-t65-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "delta", "qk_l2_norm": 1, "beta_activation": "sigmoid" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [65, 1, 20], "dtype": "float16", "dist": "normal", "seed": 59311, "scale": 0.2 }, "keyT": { "shape": [65, 1, 20], "dtype": "float16", "dist": "normal", "seed": 59312, "scale": 0.2 }, "valueT": { "shape": [65, 3, 6], "dtype": "float16", "dist": "normal", "seed": 59313, "scale": 0.2 }, "initialStateT": { "shape": [1, 3, 6, 20], "dtype": "float32", "dist": "normal", "seed": 59314, "scale": 0.05 }, "cuSeqlensT": { "shape": [2], "dtype": "int32", "data": { "kind": "values", "values": [0, 65] } }, "betaT": { "shape": [65, 3], "dtype": "float32", "dist": "normal", "seed": 59315, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [65, 3, 6] }, "finalStateT": { "dtype": "float32", "shape": [1, 3, 6, 20] } } }, { "name": "gdn-general-float16-linear-dk32-dv8-h2x6-b2-t17-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "linear", "qk_l2_norm": 1 }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [34, 2, 32], "dtype": "float16", "dist": "normal", "seed": 59322, "scale": 0.2 }, "keyT": { "shape": [34, 2, 32], "dtype": "float16", "dist": "normal", "seed": 59323, "scale": 0.2 }, "valueT": { "shape": [34, 6, 8], "dtype": "float16", "dist": "normal", "seed": 59324, "scale": 0.2 }, "initialStateT": { "shape": [2, 6, 8, 32], "dtype": "float32", "dist": "normal", "seed": 59325, "scale": 0.05 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [34, 6, 8] }, "finalStateT": { "dtype": "float32", "shape": [2, 6, 8, 32] } } }, { "name": "gdn-general-float16-gated-dk64-dv16-h2x4-b1-t129-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "gated", "qk_l2_norm": 1, "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [1, 129, 2, 64], "dtype": "float16", "dist": "normal", "seed": 59333, "scale": 0.2 }, "keyT": { "shape": [1, 129, 2, 64], "dtype": "float16", "dist": "normal", "seed": 59334, "scale": 0.2 }, "valueT": { "shape": [1, 129, 4, 16], "dtype": "float16", "dist": "normal", "seed": 59335, "scale": 0.2 }, "initialStateT": { "shape": [1, 4, 16, 64], "dtype": "float32", "dist": "normal", "seed": 59336, "scale": 0.05 }, "decayT": { "shape": [1, 129, 4], "dtype": "float32", "dist": "normal", "seed": 59337, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59338, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59339, "scale": 0.1 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [1, 129, 4, 16] }, "finalStateT": { "dtype": "float32", "shape": [1, 4, 16, 64] } } }, { "name": "gdn-general-float16-gated_delta-dk96-dv1-h1x1-b1-t33-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [33, 1, 96], "dtype": "float16", "dist": "normal", "seed": 59344, "scale": 0.2 }, "keyT": { "shape": [33, 1, 96], "dtype": "float16", "dist": "normal", "seed": 59345, "scale": 0.2 }, "valueT": { "shape": [33, 1, 1], "dtype": "float16", "dist": "normal", "seed": 59346, "scale": 0.2 }, "cuSeqlensT": { "shape": [2], "dtype": "int32", "data": { "kind": "values", "values": [0, 33] } }, "decayT": { "shape": [33, 1], "dtype": "float32", "dist": "normal", "seed": 59347, "scale": 0.1 }, "aLogT": { "shape": [1], "dtype": "float32", "dist": "normal", "seed": 59348, "scale": 0.1 }, "dtBiasT": { "shape": [1], "dtype": "float32", "dist": "normal", "seed": 59349, "scale": 0.1 }, "betaT": { "shape": [33, 1], "dtype": "float32", "dist": "normal", "seed": 59350, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [33, 1, 1] }, "finalStateT": { "dtype": "float32", "shape": [1, 1, 1, 96] } } }, { "name": "gdn-general-float16-gated_delta-dk128-dv18-h2x6-b1-t65-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [65, 2, 128], "dtype": "float16", "dist": "normal", "seed": 59355, "scale": 0.2 }, "keyT": { "shape": [65, 2, 128], "dtype": "float16", "dist": "normal", "seed": 59356, "scale": 0.2 }, "valueT": { "shape": [65, 6, 18], "dtype": "float16", "dist": "normal", "seed": 59357, "scale": 0.2 }, "initialStateT": { "shape": [1, 6, 18, 128], "dtype": "float32", "dist": "normal", "seed": 59358, "scale": 0.05 }, "cuSeqlensT": { "shape": [2], "dtype": "int32", "data": { "kind": "values", "values": [0, 65] } }, "decayT": { "shape": [65, 6], "dtype": "float32", "dist": "normal", "seed": 59359, "scale": 0.1 }, "aLogT": { "shape": [6], "dtype": "float32", "dist": "normal", "seed": 59360, "scale": 0.1 }, "dtBiasT": { "shape": [6], "dtype": "float32", "dist": "normal", "seed": 59361, "scale": 0.1 }, "betaT": { "shape": [65, 6], "dtype": "float32", "dist": "normal", "seed": 59362, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [65, 6, 18] }, "finalStateT": { "dtype": "float32", "shape": [1, 6, 18, 128] } } }, { "name": "gdn-general-float16-gated_delta-dk128-dv32-h2x6-b1-t257-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [1, 257, 2, 128], "dtype": "float16", "dist": "normal", "seed": 59366, "scale": 0.2 }, "keyT": { "shape": [1, 257, 2, 128], "dtype": "float16", "dist": "normal", "seed": 59367, "scale": 0.2 }, "valueT": { "shape": [1, 257, 6, 32], "dtype": "float16", "dist": "normal", "seed": 59368, "scale": 0.2 }, "initialStateT": { "shape": [1, 6, 32, 128], "dtype": "float32", "dist": "normal", "seed": 59369, "scale": 0.05 }, "decayT": { "shape": [1, 257, 6], "dtype": "float32", "dist": "normal", "seed": 59370, "scale": 0.1 }, "aLogT": { "shape": [6], "dtype": "float32", "dist": "normal", "seed": 59371, "scale": 0.1 }, "dtBiasT": { "shape": [6], "dtype": "float32", "dist": "normal", "seed": 59372, "scale": 0.1 }, "betaT": { "shape": [1, 257, 6], "dtype": "float32", "dist": "normal", "seed": 59373, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [1, 257, 6, 32] }, "finalStateT": { "dtype": "float32", "shape": [1, 6, 32, 128] } } }, { "name": "gdn-general-float16-gated_delta-dk256-dv64-h2x4-b2-t33-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [66, 2, 256], "dtype": "float16", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [66, 2, 256], "dtype": "float16", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [66, 4, 64], "dtype": "float16", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 256], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [66, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [66, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [66, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 256] } } }, { "name": "gdn-general-float16-linear-dk64-dv9-h1x3-b1-t65-tile4", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "linear", "qk_l2_norm": 1 }, "tunables": { "TILE_V": 4 }, "inputs": { "queryT": { "shape": [65, 1, 64], "dtype": "float16", "dist": "normal", "seed": 59388, "scale": 0.2 }, "keyT": { "shape": [65, 1, 64], "dtype": "float16", "dist": "normal", "seed": 59389, "scale": 0.2 }, "valueT": { "shape": [65, 3, 9], "dtype": "float16", "dist": "normal", "seed": 59390, "scale": 0.2 }, "cuSeqlensT": { "shape": [2], "dtype": "int32", "data": { "kind": "values", "values": [0, 65] } } }, "outputs": { "outputT": { "dtype": "float16", "shape": [65, 3, 9] }, "finalStateT": { "dtype": "float32", "shape": [1, 3, 9, 64] } } }, { "name": "gdn-general-float16-gated_delta-dk128-dv20-h2x6-b1-t33-tile4", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 4 }, "inputs": { "queryT": { "shape": [1, 33, 2, 128], "dtype": "float16", "dist": "normal", "seed": 59399, "scale": 0.2 }, "keyT": { "shape": [1, 33, 2, 128], "dtype": "float16", "dist": "normal", "seed": 59400, "scale": 0.2 }, "valueT": { "shape": [1, 33, 6, 20], "dtype": "float16", "dist": "normal", "seed": 59401, "scale": 0.2 }, "initialStateT": { "shape": [1, 6, 20, 128], "dtype": "float32", "dist": "normal", "seed": 59402, "scale": 0.05 }, "decayT": { "shape": [1, 33, 6], "dtype": "float32", "dist": "normal", "seed": 59403, "scale": 0.1 }, "aLogT": { "shape": [6], "dtype": "float32", "dist": "normal", "seed": 59404, "scale": 0.1 }, "dtBiasT": { "shape": [6], "dtype": "float32", "dist": "normal", "seed": 59405, "scale": 0.1 }, "betaT": { "shape": [1, 33, 6], "dtype": "float32", "dist": "normal", "seed": 59406, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [1, 33, 6, 20] }, "finalStateT": { "dtype": "float32", "shape": [1, 6, 20, 128] } } }, { "name": "gdn-general-float16-delta-dk32-dv12-h2x4-b1-t129-tile4", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "delta", "qk_l2_norm": 1, "beta_activation": "sigmoid" }, "tunables": { "TILE_V": 4 }, "inputs": { "queryT": { "shape": [129, 2, 32], "dtype": "float16", "dist": "normal", "seed": 59410, "scale": 0.2 }, "keyT": { "shape": [129, 2, 32], "dtype": "float16", "dist": "normal", "seed": 59411, "scale": 0.2 }, "valueT": { "shape": [129, 4, 12], "dtype": "float16", "dist": "normal", "seed": 59412, "scale": 0.2 }, "initialStateT": { "shape": [1, 4, 12, 32], "dtype": "float32", "dist": "normal", "seed": 59413, "scale": 0.05 }, "betaT": { "shape": [129, 4], "dtype": "float32", "dist": "normal", "seed": 59414, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [129, 4, 12] }, "finalStateT": { "dtype": "float32", "shape": [1, 4, 12, 32] } } }, { "name": "gdn-general-float16-gated-dk8-dv4-h1x2-b1-t65-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "gated", "qk_l2_norm": 1, "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [65, 1, 8], "dtype": "float16", "dist": "normal", "seed": 59421, "scale": 0.2 }, "keyT": { "shape": [65, 1, 8], "dtype": "float16", "dist": "normal", "seed": 59422, "scale": 0.2 }, "valueT": { "shape": [65, 2, 4], "dtype": "float16", "dist": "normal", "seed": 59423, "scale": 0.2 }, "initialStateT": { "shape": [1, 2, 4, 8], "dtype": "float32", "dist": "normal", "seed": 59424, "scale": 0.05 }, "decayT": { "shape": [65, 2], "dtype": "float32", "dist": "normal", "seed": 59425, "scale": 0.1 }, "aLogT": { "shape": [2], "dtype": "float32", "dist": "normal", "seed": 59426, "scale": 0.1 }, "dtBiasT": { "shape": [2], "dtype": "float32", "dist": "normal", "seed": 59427, "scale": 0.1 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [65, 2, 4] }, "finalStateT": { "dtype": "float32", "shape": [1, 2, 4, 8] } } }, { "name": "gdn-general-float16-gated_delta-dk128-dv32-h2x4-b1-t1-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [1, 1, 2, 128], "dtype": "float16", "dist": "normal", "seed": 59432, "scale": 0.2 }, "keyT": { "shape": [1, 1, 2, 128], "dtype": "float16", "dist": "normal", "seed": 59433, "scale": 0.2 }, "valueT": { "shape": [1, 1, 4, 32], "dtype": "float16", "dist": "normal", "seed": 59434, "scale": 0.2 }, "decayT": { "shape": [1, 1, 4], "dtype": "float32", "dist": "normal", "seed": 59435, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59436, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59437, "scale": 0.1 }, "betaT": { "shape": [1, 1, 4], "dtype": "float32", "dist": "normal", "seed": 59438, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [1, 1, 4, 32] }, "finalStateT": { "dtype": "float32", "shape": [1, 4, 32, 128] } } }, { "name": "gdn-general-float16-gated_delta-dk128-dv32-h2x4-b4-t4-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [16, 2, 128], "dtype": "float16", "dist": "normal", "seed": 59443, "scale": 0.2 }, "keyT": { "shape": [16, 2, 128], "dtype": "float16", "dist": "normal", "seed": 59444, "scale": 0.2 }, "valueT": { "shape": [16, 4, 32], "dtype": "float16", "dist": "normal", "seed": 59445, "scale": 0.2 }, "initialStateT": { "shape": [4, 4, 32, 128], "dtype": "float32", "dist": "normal", "seed": 59446, "scale": 0.05 }, "cuSeqlensT": { "shape": [5], "dtype": "int32", "data": { "kind": "values", "values": [0, 4, 8, 12, 16] } }, "decayT": { "shape": [16, 4], "dtype": "float32", "dist": "normal", "seed": 59447, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59448, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59449, "scale": 0.1 }, "betaT": { "shape": [16, 4], "dtype": "float32", "dist": "normal", "seed": 59450, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [16, 4, 32] }, "finalStateT": { "dtype": "float32", "shape": [4, 4, 32, 128] } } }, { "name": "gdn-general-float16-linear-dk64-dv64-h2x4-b1-t65-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "linear", "qk_l2_norm": 1 }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [65, 2, 64], "dtype": "float16", "dist": "normal", "seed": 59454, "scale": 0.2 }, "keyT": { "shape": [65, 2, 64], "dtype": "float16", "dist": "normal", "seed": 59455, "scale": 0.2 }, "valueT": { "shape": [65, 4, 64], "dtype": "float16", "dist": "normal", "seed": 59456, "scale": 0.2 }, "initialStateT": { "shape": [1, 4, 64, 64], "dtype": "float32", "dist": "normal", "seed": 59457, "scale": 0.05 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [65, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [1, 4, 64, 64] } } }, { "name": "gdn-general-float16-gated_delta-dk128-dv64-h2x4-b2-t129-tile2", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: unaligned head dimensions and grouped heads" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [2, 129, 2, 128], "dtype": "float16", "dist": "normal", "seed": 59465, "scale": 0.2 }, "keyT": { "shape": [2, 129, 2, 128], "dtype": "float16", "dist": "normal", "seed": 59466, "scale": 0.2 }, "valueT": { "shape": [2, 129, 4, 64], "dtype": "float16", "dist": "normal", "seed": 59467, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 128], "dtype": "float32", "dist": "normal", "seed": 59468, "scale": 0.05 }, "decayT": { "shape": [2, 129, 4], "dtype": "float32", "dist": "normal", "seed": 59469, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59470, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59471, "scale": 0.1 }, "betaT": { "shape": [2, 129, 4], "dtype": "float32", "dist": "normal", "seed": 59472, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [2, 129, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 128] } } }, { "name": "gdn-lane-density-f32-dk96-t33", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: key head-dimension sweep" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [66, 2, 96], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [66, 2, 96], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [66, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 96], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [66, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [66, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [66, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 96] } } }, { "name": "gdn-lane-density-f32-dk96-t129", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: key head-dimension sweep" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [258, 2, 96], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [258, 2, 96], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [258, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 96], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [258, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 96] } } }, { "name": "gdn-lane-density-f32-dk128-t33", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: key head-dimension sweep" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [66, 2, 128], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [66, 2, 128], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [66, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 128], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [66, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [66, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [66, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 128] } } }, { "name": "gdn-lane-density-f32-dk128-t129", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: key head-dimension sweep" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [258, 2, 128], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [258, 2, 128], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [258, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 128], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [258, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 128] } } }, { "name": "gdn-lane-density-f32-dk160-t33", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: key head-dimension sweep" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [66, 2, 160], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [66, 2, 160], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [66, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 160], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [66, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [66, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [66, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 160] } } }, { "name": "gdn-lane-density-f32-dk160-t129", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: key head-dimension sweep" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [258, 2, 160], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [258, 2, 160], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [258, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 160], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [258, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 160] } } }, { "name": "gdn-lane-density-f32-dk192-t33", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: key head-dimension sweep" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [66, 2, 192], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [66, 2, 192], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [66, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 192], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [66, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [66, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [66, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 192] } } }, { "name": "gdn-lane-density-f32-dk192-t129", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: key head-dimension sweep" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [258, 2, 192], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [258, 2, 192], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [258, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 192], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [258, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 192] } } }, { "name": "gdn-lane-density-f32-dk224-t33", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: key head-dimension sweep" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [66, 2, 224], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [66, 2, 224], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [66, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 224], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [66, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [66, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [66, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 224] } } }, { "name": "gdn-lane-density-f32-dk224-t129", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: key head-dimension sweep" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [258, 2, 224], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [258, 2, 224], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [258, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 224], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [258, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 224] } } }, { "name": "gdn-lane-density-f32-dk256-t129", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: key head-dimension sweep" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [258, 2, 256], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [258, 2, 256], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [258, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 256], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [258, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 256] } } }, { "name": "gdn-lane-density-f32-dk320-t33", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: key head-dimension sweep" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [66, 2, 320], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [66, 2, 320], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [66, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 320], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [66, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [66, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [66, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 320] } } }, { "name": "gdn-lane-density-f32-dk320-t129", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: key head-dimension sweep" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [258, 2, 320], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [258, 2, 320], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [258, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 320], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [258, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 320] } } }, { "name": "gdn-lane-density-f32-dk384-t33", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: key head-dimension sweep" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [66, 2, 384], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [66, 2, 384], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [66, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 384], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [66, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [66, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [66, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 384] } } }, { "name": "gdn-lane-density-f32-dk384-t129", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: key head-dimension sweep" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [258, 2, 384], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [258, 2, 384], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [258, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 384], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [258, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 384] } } }, { "name": "gdn-lane-density-f32-dk512-t33", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: key head-dimension sweep" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [66, 2, 512], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [66, 2, 512], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [66, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 512], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [66, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [66, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [66, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 512] } } }, { "name": "gdn-lane-density-f32-dk512-t129", "preset": "smoke", "provenance": { "source": "generic recurrence geometry: key head-dimension sweep" }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [258, 2, 512], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [258, 2, 512], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [258, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 512], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [258, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 512] } } }, { "name": "odd-key-float32-k127-t129", "preset": "smoke", "provenance": { "notes": "Measures GatedDeltaNet over key head width 127 with value head width 64, 2 query/key heads and 4 value heads, batch 2, 129 timesteps, float32 throughout." }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [258, 2, 127], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [258, 2, 127], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [258, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 127], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [258, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 127] } }, "vars": { "tokens": 258, "hv": 4, "dk": 127, "dv": 64 }, "bench": { "metrics": [{ "type": "gflops", "value": "6 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "odd-key-float32-k128-t129", "preset": "smoke", "provenance": { "notes": "Measures GatedDeltaNet over key head width 128 with value head width 64, 2 query/key heads and 4 value heads, batch 2, 129 timesteps, float32 throughout." }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [258, 2, 128], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [258, 2, 128], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [258, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 128], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [258, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 128] } }, "vars": { "tokens": 258, "hv": 4, "dk": 128, "dv": 64 }, "bench": { "metrics": [{ "type": "gflops", "value": "6 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "odd-key-float32-k129-t129", "preset": "smoke", "provenance": { "notes": "Measures GatedDeltaNet over key head width 129 with value head width 64, 2 query/key heads and 4 value heads, batch 2, 129 timesteps, float32 throughout." }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [258, 2, 129], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [258, 2, 129], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [258, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 129], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [258, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 129] } }, "vars": { "tokens": 258, "hv": 4, "dk": 129, "dv": 64 }, "bench": { "metrics": [{ "type": "gflops", "value": "6 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "odd-key-float32-k255-t129", "preset": "smoke", "provenance": { "notes": "Measures GatedDeltaNet over key head width 255 with value head width 64, 2 query/key heads and 4 value heads, batch 2, 129 timesteps, float32 throughout." }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [258, 2, 255], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [258, 2, 255], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [258, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 255], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [258, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 255] } }, "vars": { "tokens": 258, "hv": 4, "dk": 255, "dv": 64 }, "bench": { "metrics": [{ "type": "gflops", "value": "6 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "odd-key-float32-k256-t129", "preset": "smoke", "provenance": { "notes": "Measures GatedDeltaNet over key head width 256 with value head width 64, 2 query/key heads and 4 value heads, batch 2, 129 timesteps, float32 throughout." }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [258, 2, 256], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [258, 2, 256], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [258, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 256], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [258, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 256] } }, "vars": { "tokens": 258, "hv": 4, "dk": 256, "dv": 64 }, "bench": { "metrics": [{ "type": "gflops", "value": "6 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "odd-key-float32-k257-t129", "preset": "smoke", "provenance": { "notes": "Measures GatedDeltaNet over key head width 257 with value head width 64, 2 query/key heads and 4 value heads, batch 2, 129 timesteps, float32 throughout." }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [258, 2, 257], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [258, 2, 257], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [258, 4, 64], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 257], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [258, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 257] } }, "vars": { "tokens": 258, "hv": 4, "dk": 257, "dv": 64 }, "bench": { "metrics": [{ "type": "gflops", "value": "6 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "odd-key-float16-k127-t129", "preset": "smoke", "provenance": { "notes": "Measures GatedDeltaNet over key head width 127 with value head width 64, 2 query/key heads and 4 value heads, batch 2, 129 timesteps, float16 query/key/value with a float32 recurrent state." }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [258, 2, 127], "dtype": "float16", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [258, 2, 127], "dtype": "float16", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [258, 4, 64], "dtype": "float16", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 127], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [258, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 127] } }, "vars": { "tokens": 258, "hv": 4, "dk": 127, "dv": 64 }, "bench": { "metrics": [{ "type": "gflops", "value": "6 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "odd-key-float16-k128-t129", "preset": "smoke", "provenance": { "notes": "Measures GatedDeltaNet over key head width 128 with value head width 64, 2 query/key heads and 4 value heads, batch 2, 129 timesteps, float16 query/key/value with a float32 recurrent state." }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [258, 2, 128], "dtype": "float16", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [258, 2, 128], "dtype": "float16", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [258, 4, 64], "dtype": "float16", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 128], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [258, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 128] } }, "vars": { "tokens": 258, "hv": 4, "dk": 128, "dv": 64 }, "bench": { "metrics": [{ "type": "gflops", "value": "6 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "odd-key-float16-k129-t129", "preset": "smoke", "provenance": { "notes": "Measures GatedDeltaNet over key head width 129 with value head width 64, 2 query/key heads and 4 value heads, batch 2, 129 timesteps, float16 query/key/value with a float32 recurrent state." }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [258, 2, 129], "dtype": "float16", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [258, 2, 129], "dtype": "float16", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [258, 4, 64], "dtype": "float16", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 129], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [258, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 129] } }, "vars": { "tokens": 258, "hv": 4, "dk": 129, "dv": 64 }, "bench": { "metrics": [{ "type": "gflops", "value": "6 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "odd-key-float16-k255-t129", "preset": "smoke", "provenance": { "notes": "Measures GatedDeltaNet over key head width 255 with value head width 64, 2 query/key heads and 4 value heads, batch 2, 129 timesteps, float16 query/key/value with a float32 recurrent state." }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [258, 2, 255], "dtype": "float16", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [258, 2, 255], "dtype": "float16", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [258, 4, 64], "dtype": "float16", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 255], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [258, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 255] } }, "vars": { "tokens": 258, "hv": 4, "dk": 255, "dv": 64 }, "bench": { "metrics": [{ "type": "gflops", "value": "6 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "odd-key-float16-k256-t129", "preset": "smoke", "provenance": { "notes": "Measures GatedDeltaNet over key head width 256 with value head width 64, 2 query/key heads and 4 value heads, batch 2, 129 timesteps, float16 query/key/value with a float32 recurrent state." }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [258, 2, 256], "dtype": "float16", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [258, 2, 256], "dtype": "float16", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [258, 4, 64], "dtype": "float16", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 256], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [258, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 256] } }, "vars": { "tokens": 258, "hv": 4, "dk": 256, "dv": 64 }, "bench": { "metrics": [{ "type": "gflops", "value": "6 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "odd-key-float16-k257-t129", "preset": "smoke", "provenance": { "notes": "Measures GatedDeltaNet over key head width 257 with value head width 64, 2 query/key heads and 4 value heads, batch 2, 129 timesteps, float16 query/key/value with a float32 recurrent state." }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [258, 2, 257], "dtype": "float16", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [258, 2, 257], "dtype": "float16", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [258, 4, 64], "dtype": "float16", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 64, 257], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [258, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [258, 4, 64] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 64, 257] } }, "vars": { "tokens": 258, "hv": 4, "dk": 257, "dv": 64 }, "bench": { "metrics": [{ "type": "gflops", "value": "6 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "density-neighbor-float32-k33-t65", "preset": "smoke", "provenance": { "notes": "Measures GatedDeltaNet over key head width 33 with value head width 5, 2 query/key heads and 4 value heads, batch 2, 65 timesteps, float32 throughout." }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [130, 2, 33], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [130, 2, 33], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [130, 4, 5], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 5, 33], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [130, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [130, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [130, 4, 5] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 5, 33] } }, "vars": { "tokens": 130, "hv": 4, "dk": 33, "dv": 5 }, "bench": { "metrics": [{ "type": "gflops", "value": "6 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "density-neighbor-float32-k65-t65", "preset": "smoke", "provenance": { "notes": "Measures GatedDeltaNet over key head width 65 with value head width 5, 2 query/key heads and 4 value heads, batch 2, 65 timesteps, float32 throughout." }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [130, 2, 65], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [130, 2, 65], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [130, 4, 5], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 5, 65], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [130, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [130, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [130, 4, 5] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 5, 65] } }, "vars": { "tokens": 130, "hv": 4, "dk": 65, "dv": 5 }, "bench": { "metrics": [{ "type": "gflops", "value": "6 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "density-neighbor-float32-k511-t65", "preset": "smoke", "provenance": { "notes": "Measures GatedDeltaNet over key head width 511 with value head width 5, 2 query/key heads and 4 value heads, batch 2, 65 timesteps, float32 throughout." }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [130, 2, 511], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [130, 2, 511], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [130, 4, 5], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 5, 511], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [130, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [130, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [130, 4, 5] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 5, 511] } }, "vars": { "tokens": 130, "hv": 4, "dk": 511, "dv": 5 }, "bench": { "metrics": [{ "type": "gflops", "value": "6 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "density-neighbor-float32-k513-t65", "preset": "smoke", "provenance": { "notes": "Measures GatedDeltaNet over key head width 513 with value head width 5, 2 query/key heads and 4 value heads, batch 2, 65 timesteps, float32 throughout." }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [130, 2, 513], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [130, 2, 513], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [130, 4, 5], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 5, 513], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [130, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [130, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [130, 4, 5] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 5, 513] } }, "vars": { "tokens": 130, "hv": 4, "dk": 513, "dv": 5 }, "bench": { "metrics": [{ "type": "gflops", "value": "6 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "density-neighbor-float16-k33-t65", "preset": "smoke", "provenance": { "notes": "Measures GatedDeltaNet over key head width 33 with value head width 5, 2 query/key heads and 4 value heads, batch 2, 65 timesteps, float16 query/key/value with a float32 recurrent state." }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [130, 2, 33], "dtype": "float16", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [130, 2, 33], "dtype": "float16", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [130, 4, 5], "dtype": "float16", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 5, 33], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [130, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [130, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [130, 4, 5] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 5, 33] } }, "vars": { "tokens": 130, "hv": 4, "dk": 33, "dv": 5 }, "bench": { "metrics": [{ "type": "gflops", "value": "6 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "density-neighbor-float16-k65-t65", "preset": "smoke", "provenance": { "notes": "Measures GatedDeltaNet over key head width 65 with value head width 5, 2 query/key heads and 4 value heads, batch 2, 65 timesteps, float16 query/key/value with a float32 recurrent state." }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [130, 2, 65], "dtype": "float16", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [130, 2, 65], "dtype": "float16", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [130, 4, 5], "dtype": "float16", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 5, 65], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [130, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [130, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [130, 4, 5] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 5, 65] } }, "vars": { "tokens": 130, "hv": 4, "dk": 65, "dv": 5 }, "bench": { "metrics": [{ "type": "gflops", "value": "6 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "density-neighbor-float16-k511-t65", "preset": "smoke", "provenance": { "notes": "Measures GatedDeltaNet over key head width 511 with value head width 5, 2 query/key heads and 4 value heads, batch 2, 65 timesteps, float16 query/key/value with a float32 recurrent state." }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [130, 2, 511], "dtype": "float16", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [130, 2, 511], "dtype": "float16", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [130, 4, 5], "dtype": "float16", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 5, 511], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [130, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [130, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [130, 4, 5] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 5, 511] } }, "vars": { "tokens": 130, "hv": 4, "dk": 511, "dv": 5 }, "bench": { "metrics": [{ "type": "gflops", "value": "6 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "density-neighbor-float16-k513-t65", "preset": "smoke", "provenance": { "notes": "Measures GatedDeltaNet over key head width 513 with value head width 5, 2 query/key heads and 4 value heads, batch 2, 65 timesteps, float16 query/key/value with a float32 recurrent state." }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [130, 2, 513], "dtype": "float16", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [130, 2, 513], "dtype": "float16", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [130, 4, 5], "dtype": "float16", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 5, 513], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [130, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [130, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [130, 4, 5] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 5, 513] } }, "vars": { "tokens": 130, "hv": 4, "dk": 513, "dv": 5 }, "bench": { "metrics": [{ "type": "gflops", "value": "6 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "density-capacity-float32-k1023-t65", "preset": "smoke", "provenance": { "notes": "Measures GatedDeltaNet over key head width 1023 with value head width 5, 2 query/key heads and 4 value heads, batch 2, 65 timesteps, float32 throughout." }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [130, 2, 1023], "dtype": "float32", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [130, 2, 1023], "dtype": "float32", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [130, 4, 5], "dtype": "float32", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 5, 1023], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [130, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [130, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float32", "shape": [130, 4, 5] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 5, 1023] } }, "vars": { "tokens": 130, "hv": 4, "dk": 1023, "dv": 5 }, "bench": { "metrics": [{ "type": "gflops", "value": "6 * args.tokens * args.hv * args.dk * args.dv" }] } }, { "name": "density-capacity-float16-k1023-t65", "preset": "smoke", "provenance": { "notes": "Measures GatedDeltaNet over key head width 1023 with value head width 5, 2 query/key heads and 4 value heads, batch 2, 65 timesteps, float16 query/key/value with a float32 recurrent state." }, "attrs": { "update_rule": "gated_delta", "qk_l2_norm": 1, "beta_activation": "sigmoid", "gate_activation": "qwen" }, "tunables": { "TILE_V": 2 }, "inputs": { "queryT": { "shape": [130, 2, 1023], "dtype": "float16", "dist": "normal", "seed": 59377, "scale": 0.2 }, "keyT": { "shape": [130, 2, 1023], "dtype": "float16", "dist": "normal", "seed": 59378, "scale": 0.2 }, "valueT": { "shape": [130, 4, 5], "dtype": "float16", "dist": "normal", "seed": 59379, "scale": 0.2 }, "initialStateT": { "shape": [2, 4, 5, 1023], "dtype": "float32", "dist": "normal", "seed": 59380, "scale": 0.05 }, "decayT": { "shape": [130, 4], "dtype": "float32", "dist": "normal", "seed": 59381, "scale": 0.1 }, "aLogT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59382, "scale": 0.1 }, "dtBiasT": { "shape": [4], "dtype": "float32", "dist": "normal", "seed": 59383, "scale": 0.1 }, "betaT": { "shape": [130, 4], "dtype": "float32", "dist": "normal", "seed": 59384, "scale": 0.5 } }, "outputs": { "outputT": { "dtype": "float16", "shape": [130, 4, 5] }, "finalStateT": { "dtype": "float32", "shape": [2, 4, 5, 1023] } }, "vars": { "tokens": 130, "hv": 4, "dk": 1023, "dv": 5 }, "bench": { "metrics": [{ "type": "gflops", "value": "6 * args.tokens * args.hv * args.dk * args.dv" }] } } ] }