Hugging Face's logo

Kernels:
webgpu-kernels
/
ai.onnx.QLinearConv

Kernel card Files
xet
 Community
Xenova HF Staff commited on
Commit
396ebd7
·
verified ·
1 Parent(s): 7a72eee

sync 6fdf6301e2bb

Browse files
Files changed (12) hide show
  1. README.md +2 -2
  2. build/webgpu/bench.json +1 -1
  3. build/webgpu/conv-int-accumulate-spatial.wgsl.jinja +39 -54
  4. build/webgpu/conv-int-im2col-spatial.wgsl.jinja +29 -20
  5. build/webgpu/manifest.json +120 -349
  6. build/webgpu/metadata.json +13 -13
  7. build/webgpu/qlinear-conv-dp4a-direct-3x3.wgsl.jinja +264 -9
  8. build/webgpu/qlinear-conv-nchw-x4.wgsl.jinja +267 -45
  9. build/webgpu/qlinear-conv-requantize.wgsl.jinja +268 -42
  10. build/webgpu/quant-dp4a-matmul.wgsl.jinja +286 -39
  11. build/webgpu/quant-pad-k-matrices.wgsl.jinja +9 -7
  12. build/webgpu/test.json +102 -13
README.md CHANGED
@@ -63,7 +63,7 @@ Attributes and default values (overridable per request):
63
  - [`metadata.json`](build/webgpu/metadata.json) — kernel metadata (id, digests, per-variant templates, provenance)
64
  - [`manifest.json`](build/webgpu/manifest.json) — the op contract (source of truth)
65
  - [`test.json`](build/webgpu/test.json) — correctness cases
66
- - [`bench.json`](build/webgpu/bench.json) — benchmark + tuning cases
67
  - [`conv-int-accumulate-spatial.wgsl.jinja`](build/webgpu/conv-int-accumulate-spatial.wgsl.jinja)
68
  - [`conv-int-im2col-spatial.wgsl.jinja`](build/webgpu/conv-int-im2col-spatial.wgsl.jinja)
69
  - [`qlinear-conv-dp4a-direct-3x3.wgsl.jinja`](build/webgpu/qlinear-conv-dp4a-direct-3x3.wgsl.jinja)
@@ -75,7 +75,7 @@ Attributes and default values (overridable per request):
75
  ## Use with `@huggingface/kernels`
76
 
77
  ```sh
78
- npm install --save-exact @huggingface/kernels@0.0.1-preview.2
79
  ```
80
 
81
  Required output shapes and logical data types are inferred from the supplied inputs and attributes; result tensors are allocated automatically.
 
63
  - [`metadata.json`](build/webgpu/metadata.json) — kernel metadata (id, digests, per-variant templates, provenance)
64
  - [`manifest.json`](build/webgpu/manifest.json) — the op contract (source of truth)
65
  - [`test.json`](build/webgpu/test.json) — correctness cases
66
+ - [`bench.json`](build/webgpu/bench.json) — benchmark cases
67
  - [`conv-int-accumulate-spatial.wgsl.jinja`](build/webgpu/conv-int-accumulate-spatial.wgsl.jinja)
68
  - [`conv-int-im2col-spatial.wgsl.jinja`](build/webgpu/conv-int-im2col-spatial.wgsl.jinja)
69
  - [`qlinear-conv-dp4a-direct-3x3.wgsl.jinja`](build/webgpu/qlinear-conv-dp4a-direct-3x3.wgsl.jinja)
 
75
  ## Use with `@huggingface/kernels`
76
 
77
  ```sh
78
+ npm install --save-exact @huggingface/kernels@0.0.1-preview.3
79
  ```
80
 
81
  Required output shapes and logical data types are inferred from the supplied inputs and attributes; result tensors are allocated automatically.
build/webgpu/bench.json CHANGED
@@ -137,7 +137,7 @@
137
  }
138
  },
139
  {
140
- "name": "bias3x3_forces_naive_accumulate_1x128x64x64_oc128_u8s8",
141
  "preset": "stress",
142
  "vars": { "batch": 1, "inChannels": 128, "outChannels": 128, "inH": 64, "inW": 64, "kernelH": 3, "kernelW": 3 },
143
  "attrs": { "strides": [1, 1], "pads": [1, 1, 1, 1] },
 
137
  }
138
  },
139
  {
140
+ "name": "bias3x3_naive_accumulate_1x128x64x64_oc128_u8s8",
141
  "preset": "stress",
142
  "vars": { "batch": 1, "inChannels": 128, "outChannels": 128, "inH": 64, "inW": 64, "kernelH": 3, "kernelW": 3 },
143
  "attrs": { "strides": [1, 1], "pads": [1, 1, 1, 1] },
build/webgpu/conv-int-accumulate-spatial.wgsl.jinja CHANGED
@@ -1,36 +1,41 @@
1
- {% macro flat_index_2d(name="i", bound="params.count", guardInline=false, note="dispatch-limit") %}
2
- {% if note == "dispatch-limit" %}
3
- // 2D-folded flat index: gid.y carries the high bits past the dispatch's
4
- // per-axis workgroup fold width (outputs > 16.7M elements).
5
- {% elif note == "limit" %}
6
- // 2D-folded flat index: gid.y carries the high bits past the dispatch's
7
- // per-axis workgroup fold width.
8
- {% elif note == "device-axis" %}
9
- // The flat dispatch is folded across x/y at a fixed per-axis workgroup
10
- // width; gid.y carries the high portion of the output index.
11
- {% elif note == "vec4-limit" %}
12
- // 2D-folded flat vec4 index: gid.y carries the high bits past the dispatch's
13
- // per-axis workgroup fold width (the dispatch caps x and spills into y).
14
- {% elif note == "element-limit" %}
15
- // 2D-folded flat element index: gid.y carries the high bits past the
16
- // dispatch's per-axis workgroup fold width.
17
- {% elif note == "dispatch" %}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18
  // 2D-folded flat index: gid.y carries the high bits past the dispatch's
19
  // per-axis workgroup fold width.
20
- {% endif %}
21
- {% if bound == "" %}
22
- let {{ name }} = gid.x + gid.y * {{ DISPATCH_FOLD_WIDTH }}u * {{ tunables.WORKGROUP_SIZE }}u;
23
- {%- elif guardInline %}
24
- let {{ name }} = gid.x + gid.y * {{ DISPATCH_FOLD_WIDTH }}u * {{ tunables.WORKGROUP_SIZE }}u;
25
- if ({{ name }} >= {{ bound }}) { return; }
26
- {%- else %}
27
- let {{ name }} = gid.x + gid.y * {{ DISPATCH_FOLD_WIDTH }}u * {{ tunables.WORKGROUP_SIZE }}u;
28
- if ({{ name }} >= {{ bound }}) {
29
- return;
30
- }
31
- {%- endif %}
32
- {% endmacro %}
33
-
34
  {{ env.wgsl.resourceDeclarations }}
35
 
36
  fn read_x(index: u32) -> i32 {
@@ -49,34 +54,14 @@ fn read_w(index: u32) -> i32 {
49
  {% endif %}
50
  }
51
 
52
- fn read_x_zero() -> i32 {
53
- {% if xUnsigned %}
54
- return i32(x_zero_point[0]);
55
- {% else %}
56
- return x_zero_point[0];
57
- {% endif %}
58
- }
59
 
60
  // A per-output-channel weight zero point applies at every spatial rank.
61
- fn read_w_zero({% if wZeroPerChannel %}oc: u32{% endif %}) -> i32 {
62
- {% if wZeroPerChannel %}
63
- {% if wUnsigned %}
64
- return i32(w_zero_point[oc]);
65
- {% else %}
66
- return w_zero_point[oc];
67
- {% endif %}
68
- {% else %}
69
- {% if wUnsigned %}
70
- return i32(w_zero_point[0]);
71
- {% else %}
72
- return w_zero_point[0];
73
- {% endif %}
74
- {% endif %}
75
- }
76
 
77
  @compute @workgroup_size({{ tunables.WORKGROUP_SIZE }})
78
  fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
79
- {{ flat_index_2d("index", guardInline=true) }}
80
  let ow = index % params.outW;
81
  var t = index / params.outW;
82
  let oh = t % params.outH;
 
1
+ {% macro quant_read_scalar_zero(fn_name, buffer_name, is_unsigned, present=true) %}
2
+ fn {{ fn_name }}() -> i32 {
3
+ {% if present %}
4
+ {% if is_unsigned %}
5
+ return i32({{ buffer_name }}[0]);
6
+ {% else %}
7
+ return {{ buffer_name }}[0];
8
+ {% endif %}
9
+ {% else %}
10
+ return 0;
11
+ {% endif %}
12
+ }{% endmacro %}
13
+ {% macro quant_read_column_zero(fn_name, buffer_name, is_unsigned, present=true, per_column=false, param_name="col", return_type="i32") %}
14
+ fn {{ fn_name }}({% if per_column %}{{ param_name }}: u32{% endif %}) -> {{ return_type }} {
15
+ {% if present %}
16
+ {% if per_column %}
17
+ {% if is_unsigned %}
18
+ return i32({{ buffer_name }}[{{ param_name }}]);
19
+ {% else %}
20
+ return {{ buffer_name }}[{{ param_name }}];
21
+ {% endif %}
22
+ {% else %}
23
+ {% if is_unsigned %}
24
+ return i32({{ buffer_name }}[0]);
25
+ {% else %}
26
+ return {{ buffer_name }}[0];
27
+ {% endif %}
28
+ {% endif %}
29
+ {% else %}
30
+ return 0;
31
+ {% endif %}
32
+ }{% endmacro %}
33
+ {% macro flat_index_2d(workgroupSize, name="i", bound="params.count", guardInline=false) %}
34
+ {% set wgTerm = workgroupSize ~ "u" if workgroupSize is number else workgroupSize %}
35
  // 2D-folded flat index: gid.y carries the high bits past the dispatch's
36
  // per-axis workgroup fold width.
37
+ let {{ name }} = gid.x + gid.y * {{ DISPATCH_FOLD_WIDTH }}u * {{ wgTerm }};
38
+ if ({{ name }} >= {{ bound }}) { return; }{% endmacro %}
 
 
 
 
 
 
 
 
 
 
 
 
39
  {{ env.wgsl.resourceDeclarations }}
40
 
41
  fn read_x(index: u32) -> i32 {
 
54
  {% endif %}
55
  }
56
 
57
+ {{ quant_read_scalar_zero("read_x_zero", "x_zero_point", xUnsigned, not (xZeroOmitted is defined and xZeroOmitted)) }}
 
 
 
 
 
 
58
 
59
  // A per-output-channel weight zero point applies at every spatial rank.
60
+ {{ quant_read_column_zero("read_w_zero", "w_zero_point", wUnsigned, not (wZeroOmitted is defined and wZeroOmitted), wZeroPerChannel, "oc") }}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
61
 
62
  @compute @workgroup_size({{ tunables.WORKGROUP_SIZE }})
63
  fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
64
+ {{ flat_index_2d(tunables.WORKGROUP_SIZE, "index", guardInline=true) }}
65
  let ow = index % params.outW;
66
  var t = index / params.outW;
67
  let oh = t % params.outH;
build/webgpu/conv-int-im2col-spatial.wgsl.jinja CHANGED
@@ -6,6 +6,11 @@
6
  * reduction. */
7
  {{ env.wgsl.resourceDeclarations }}
8
  {% set padK = padK is defined and padK %}
 
 
 
 
 
9
  {% if spatialRankSpec == 3 %}
10
 
11
  const KERNEL_D: u32 = {{ kernelDSpec }}u;
@@ -27,23 +32,10 @@ const DILATION_H: u32 = {{ dilationHSpec }}u;
27
  const DILATION_W: u32 = {{ dilationWSpec }}u;
28
  const PAD_TOP: i32 = {{ padTopSpec }};
29
  const PAD_LEFT: i32 = {{ padLeftSpec }};
30
-
31
- @compute @workgroup_size({{ tunables.WORKGROUP_SIZE }})
32
- fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
33
- let pos = gid.x;
34
- if (pos >= params.outCount) {
35
- return;
36
- }
37
- let k = gid.y;
38
- let batch = gid.z;
39
- {% if padK %}
40
- if (k >= params.sourceKRows) {
41
- cols[(batch * params.kRows + k) * params.outCount + pos] = x_zero_point[0];
42
- return;
43
- }
44
- {% endif %}
45
- let ic = k / KSIZE;
46
- let kq = k - ic * KSIZE;
47
  {% if spatialRankSpec == 3 %}
48
  let kd = kq / KERNEL_HW;
49
  let khw = kq - kd * KERNEL_HW;
@@ -63,22 +55,39 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
63
  {% endif %}
64
  let ih = i32(oh * STRIDE_H + kh * DILATION_H) - PAD_TOP;
65
  let iw = i32(ow * STRIDE_W + kw * DILATION_W) - PAD_LEFT;
66
- var value: {{ bScalar }} = x_zero_point[0];
67
  {% if spatialRankSpec == 3 %}
68
  if (
69
  id >= 0 && id < i32(params.inD)
70
  && ih >= 0 && ih < i32(params.inH)
71
  && iw >= 0 && iw < i32(params.inW)
72
  ) {
73
- value = x[
74
  (((batch * params.inChannels + ic) * params.inD + u32(id)) * params.inH + u32(ih))
75
  * params.inW + u32(iw)
76
  ];
77
  }
78
  {% else %}
79
  if (ih >= 0 && ih < i32(params.inH) && iw >= 0 && iw < i32(params.inW)) {
80
- value = x[((batch * params.inChannels + ic) * params.inH + u32(ih)) * params.inW + u32(iw)];
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
81
  }
82
  {% endif %}
 
83
  cols[(batch * params.kRows + k) * params.outCount + pos] = value;
84
  }
 
6
  * reduction. */
7
  {{ env.wgsl.resourceDeclarations }}
8
  {% set padK = padK is defined and padK %}
9
+ {% set kernelDSpec = kernelDSpec | default(1) %}
10
+ {% set strideDSpec = strideDSpec | default(1) %}
11
+ {% set dilationDSpec = dilationDSpec | default(1) %}
12
+ {% set padFrontSpec = padFrontSpec | default(0) %}
13
+ {% macro spatial_constants() %}
14
  {% if spatialRankSpec == 3 %}
15
 
16
  const KERNEL_D: u32 = {{ kernelDSpec }}u;
 
32
  const DILATION_W: u32 = {{ dilationWSpec }}u;
33
  const PAD_TOP: i32 = {{ padTopSpec }};
34
  const PAD_LEFT: i32 = {{ padLeftSpec }};
35
+ {% endmacro %}
36
+ {% macro spatial_sample(value="value", input="x", zero="x_zero_point", sampleK="k", sampleScalar="auto") %}
37
+ let ic = {{ sampleK }} / KSIZE;
38
+ let kq = {{ sampleK }} - ic * KSIZE;
 
 
 
 
 
 
 
 
 
 
 
 
 
39
  {% if spatialRankSpec == 3 %}
40
  let kd = kq / KERNEL_HW;
41
  let khw = kq - kd * KERNEL_HW;
 
55
  {% endif %}
56
  let ih = i32(oh * STRIDE_H + kh * DILATION_H) - PAD_TOP;
57
  let iw = i32(ow * STRIDE_W + kw * DILATION_W) - PAD_LEFT;
58
+ var {{ value }}: {{ bScalar if sampleScalar == "auto" else sampleScalar }} = {{ zero }}[0];
59
  {% if spatialRankSpec == 3 %}
60
  if (
61
  id >= 0 && id < i32(params.inD)
62
  && ih >= 0 && ih < i32(params.inH)
63
  && iw >= 0 && iw < i32(params.inW)
64
  ) {
65
+ {{ value }} = {{ input }}[
66
  (((batch * params.inChannels + ic) * params.inD + u32(id)) * params.inH + u32(ih))
67
  * params.inW + u32(iw)
68
  ];
69
  }
70
  {% else %}
71
  if (ih >= 0 && ih < i32(params.inH) && iw >= 0 && iw < i32(params.inW)) {
72
+ {{ value }} = {{ input }}[((batch * params.inChannels + ic) * params.inH + u32(ih)) * params.inW + u32(iw)];
73
+ }
74
+ {% endif %}
75
+ {% endmacro %}
76
+ {{ spatial_constants() }}
77
+ @compute @workgroup_size({{ tunables.WORKGROUP_SIZE }})
78
+ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
79
+ let pos = gid.x;
80
+ if (pos >= params.outCount) {
81
+ return;
82
+ }
83
+ let k = gid.y;
84
+ let batch = gid.z;
85
+ {% if padK %}
86
+ if (k >= params.sourceKRows) {
87
+ cols[(batch * params.kRows + k) * params.outCount + pos] = x_zero_point[0];
88
+ return;
89
  }
90
  {% endif %}
91
+ {{ spatial_sample() }}
92
  cols[(batch * params.kRows + k) * params.outCount + pos] = value;
93
  }
build/webgpu/manifest.json CHANGED
@@ -90,14 +90,27 @@
90
  "explicitPadsOk": "attrs.auto_pad == \"NOTSET\" or not has(attrs, \"pads\")",
91
  "spatialAttributesOk": "spatialRank >= 1 and spatialRank <= 3 and spatialAttributeLengthsOk and kernelShapeMatchesWeights and kernelExtentsOk and stridesValuesOk and dilationsValuesOk and padsValuesOk and explicitPadsOk",
92
  "packedFeature": "device.wgslLanguageFeatures.has(\"packed_4x8_integer_dot_product\")",
93
- "xyQuantContract": "(ranks.x_scale == 0 or (ranks.x_scale == 1 and dim(shapes.x_scale, 0) == 1)) and (ranks.y_scale == 0 or (ranks.y_scale == 1 and dim(shapes.y_scale, 0) == 1)) and (ranks.x_zero_point == 0 or (ranks.x_zero_point == 1 and dim(shapes.x_zero_point, 0) == 1)) and (ranks.y_zero_point == 0 or (ranks.y_zero_point == 1 and dim(shapes.y_zero_point, 0) == 1))",
94
- "rank3TensorOk": "spatialAttributesOk and ranks.x == 3 and ranks.w == 3 and ranks.y == 3",
95
  "rank4TensorOk": "spatialAttributesOk and (ranks.x == 4 and ranks.w == 4 and ranks.y == 4)",
96
- "rank5TensorOk": "spatialAttributesOk and ranks.x == 5 and ranks.w == 5 and ranks.y == 5",
97
  "outputBatchChannelsOk": "dim(shapes.y, 0) == dim(shapes.x, 0) and dim(shapes.y, 1) == dim(shapes.w, 0)",
98
  "groupChannelsOk": "attrs.group >= 1 and dim(shapes.w, 1) * attrs.group == dim(shapes.x, 1) and dim(shapes.w, 0) % attrs.group == 0",
99
  "output2dShapeOk": "dim(shapes.y, 2) == expectedOutputHeight and dim(shapes.y, 3) == expectedOutputWidth",
 
 
 
 
 
 
 
 
 
 
 
 
100
  "scalarWeightQuantOk": "(ranks.w_scale == 0 or (ranks.w_scale == 1 and dim(shapes.w_scale, 0) == 1)) and (ranks.w_zero_point == 0 or (ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == 1))",
 
 
 
 
101
  "flexibleWeightQuantOk": "(((ranks.w_scale == 0 or (ranks.w_scale == 1 and dim(shapes.w_scale, 0) == 1)) and (ranks.w_zero_point == 0 or (ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == 1))) or (ranks.w_scale == 1 and dim(shapes.w_scale, 0) == dim(shapes.w, 0) and ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == dim(shapes.w, 0)))",
102
  "dp4a2dDispatchOk": "ceilDiv(dim(shapes.y, 2) * dim(shapes.y, 3), 64) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(dim(shapes.w, 0), 8) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and dim(shapes.y, 0) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)",
103
  "convM": "dim(shapes.w, 0)",
@@ -112,22 +125,26 @@
112
  "paddedPreparationDispatchOk": "ceilDiv(inner(shapes.y, 1), tunables.WORKGROUP_SIZE) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(dim(shapes.w, 1) * inner(shapes.w, 1), 4) * 4 <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(dim(shapes.w, 0) * ceilDiv(dim(shapes.w, 1) * inner(shapes.w, 1), 4) * 4, tunables.WORKGROUP_SIZE) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)",
113
  "direct3x3SharedBytes": "4 * ((tunables.DP4A_DIRECT_IC_TILE / 4) * (tunables.DP4A_DIRECT_SPATIAL_TILE + 2) * (tunables.DP4A_DIRECT_SPATIAL_TILE + 2) + tunables.DP4A_DIRECT_OC_TILE * (tunables.DP4A_DIRECT_IC_TILE / 4) * 9 + tunables.DP4A_DIRECT_OC_TILE)",
114
  "direct3x3DispatchOk": "ceilDiv(dim(shapes.y, 3), tunables.DP4A_DIRECT_SPATIAL_TILE) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(dim(shapes.y, 2), tunables.DP4A_DIRECT_SPATIAL_TILE) * ceilDiv(dim(shapes.w, 0), tunables.DP4A_DIRECT_OC_TILE) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and dim(shapes.y, 0) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)",
115
- "direct3x3DeviceOk": "tunables.DP4A_DIRECT_IC_TILE % 4 == 0 and tunables.DP4A_DIRECT_SPATIAL_TILE * tunables.DP4A_DIRECT_SPATIAL_TILE <= device.limits.maxComputeInvocationsPerWorkgroup and tunables.DP4A_DIRECT_SPATIAL_TILE <= device.limits.maxComputeWorkgroupSizeX and tunables.DP4A_DIRECT_SPATIAL_TILE <= device.limits.maxComputeWorkgroupSizeY and direct3x3SharedBytes <= device.limits.maxComputeWorkgroupStorageSize and direct3x3DispatchOk"
 
 
 
 
 
116
  },
117
  "when": ["outputBatchChannelsOk", "xyQuantContract"],
118
  "bindings": {
119
- "a": { "arg": "w", "buffer": "read-only-storage", "elementType": "i32" },
120
- "a_zero_point": { "arg": "w_zero_point", "buffer": "read-only-storage", "elementType": "i32" },
121
- "b_zero_point": { "arg": "x_zero_point", "buffer": "read-only-storage", "elementType": "u32", "length": 1 },
122
- "a_padded": { "scratch": "w_padded", "buffer": "storage", "elementType": "i32" },
123
- "a_2": { "scratch": "w_padded", "name": "a", "buffer": "read-only-storage", "elementType": "vec4<i32>" },
124
- "a_scale": { "arg": "x_scale", "buffer": "read-only-storage", "elementType": "f32", "length": 1 },
125
- "b_scale": { "arg": "w_scale", "buffer": "read-only-storage", "elementType": "f32" },
126
- "y_scale": { "buffer": "read-only-storage", "elementType": "f32", "length": 1 },
127
- "y_zero_point": { "buffer": "read-only-storage", "elementType": "u32", "length": 1 },
128
- "y": { "buffer": "storage", "elementType": "u32" },
129
  "params": {
130
- "buffer": "uniform",
131
  "struct": [
132
  { "name": "M", "type": "u32", "value": "convM" },
133
  { "name": "N", "type": "u32", "value": "convN" },
@@ -137,32 +154,20 @@
137
  { "name": "yBatchStride", "type": "u32", "value": "convYStride" }
138
  ]
139
  },
140
- "x": { "buffer": "read-only-storage", "elementType": "u32" },
141
- "x_zero_point": { "buffer": "read-only-storage", "elementType": "u32", "length": 1 },
142
- "cols": { "scratch": "cols_padded", "buffer": "storage", "elementType": "u32" },
143
- "x_scale": { "buffer": "read-only-storage", "elementType": "f32", "length": 1 },
144
- "w": { "buffer": "read-only-storage", "elementType": "i32" },
145
- "w_scale": { "buffer": "read-only-storage", "elementType": "f32" },
146
- "w_zero_point": { "buffer": "read-only-storage", "elementType": "i32" },
147
- "a_3": { "arg": "w", "name": "a", "buffer": "read-only-storage", "elementType": "$aVec4" },
148
- "a_zero_point_2": {
149
- "arg": "w_zero_point",
150
- "name": "a_zero_point",
151
- "buffer": "read-only-storage",
152
- "elementType": "$aScalar"
153
- },
154
- "b_zero_point_2": {
155
- "arg": "x_zero_point",
156
- "name": "b_zero_point",
157
- "buffer": "read-only-storage",
158
- "elementType": "$bScalar",
159
- "length": 1
160
- },
161
- "y_zero_point_2": { "name": "y_zero_point", "buffer": "read-only-storage", "elementType": "$yScalar", "length": 1 },
162
- "y_2": { "name": "y", "buffer": "storage", "elementType": "$yScalar" },
163
- "params_5": {
164
  "name": "params",
165
- "buffer": "uniform",
166
  "struct": [
167
  { "name": "M", "type": "u32", "value": "convM" },
168
  { "name": "N", "type": "u32", "value": "convN" },
@@ -172,14 +177,13 @@
172
  { "name": "yBatchStride", "type": "u32", "value": "convYStride" }
173
  ]
174
  },
175
- "x_2": { "name": "x", "buffer": "read-only-storage", "elementType": "$xScalar" },
176
- "x_zero_point_2": { "name": "x_zero_point", "buffer": "read-only-storage", "elementType": "$xScalar", "length": 1 },
177
- "w_2": { "name": "w", "buffer": "read-only-storage", "elementType": "$wScalar" },
178
- "w_zero_point_2": { "name": "w_zero_point", "buffer": "read-only-storage", "elementType": "$wScalar" },
179
- "y_3": { "scratch": "accum", "name": "y", "buffer": "storage", "elementType": "i32" },
180
- "params_7": {
181
  "name": "params",
182
- "buffer": "uniform",
183
  "struct": [
184
  { "name": "inChannels", "type": "u32", "value": "dim(shapes.x, 1)" },
185
  { "name": "inD", "type": "u32", "value": "dim(shapes.x, 2)" },
@@ -207,9 +211,8 @@
207
  { "name": "count", "type": "u32", "value": "numel(shapes.y)" }
208
  ]
209
  },
210
- "params_8": {
211
  "name": "params",
212
- "buffer": "uniform",
213
  "struct": [
214
  { "name": "inChannels", "type": "u32", "value": "dim(shapes.x, 1)" },
215
  { "name": "inH", "type": "u32", "value": 1 },
@@ -231,9 +234,8 @@
231
  { "name": "count", "type": "u32", "value": "numel(shapes.y)" }
232
  ]
233
  },
234
- "params_9": {
235
  "name": "params",
236
- "buffer": "uniform",
237
  "struct": [
238
  { "name": "inChannels", "type": "u32", "value": "dim(shapes.x, 1)" },
239
  { "name": "inH", "type": "u32", "value": "dim(shapes.x, 2)" },
@@ -256,27 +258,21 @@
256
  ]
257
  },
258
  "accum": { "buffer": "read-only-storage", "elementType": "i32" },
259
- "params_10": {
260
  "name": "params",
261
- "buffer": "uniform",
262
  "struct": [
263
  { "name": "count", "type": "u32", "value": "numel(shapes.y)" },
264
  { "name": "outChannels", "type": "u32", "value": "dim(shapes.w, 0)" },
265
  { "name": "outImageSize", "type": "u32", "value": "numel(shapes.y) / (dim(shapes.y, 0) * dim(shapes.y, 1))" }
266
  ]
267
  },
268
- "params_11": {
269
- "name": "params",
270
- "buffer": "uniform",
271
- "struct": [{ "name": "count", "type": "u32", "value": "numel(shapes.y)" }]
272
- },
273
- "B": { "buffer": "read-only-storage", "elementType": "i32" },
274
- "x_3": { "name": "x", "buffer": "read-only-storage", "elementType": "$bScalar" },
275
- "x_zero_point_3": { "name": "x_zero_point", "buffer": "read-only-storage", "elementType": "$bScalar", "length": 1 },
276
- "cols_2": { "scratch": "cols3d", "name": "cols", "buffer": "storage", "elementType": "$bScalar" },
277
- "params_12": {
278
  "name": "params",
279
- "buffer": "uniform",
280
  "struct": [
281
  { "name": "outCount", "type": "u32", "value": "dim(shapes.y, 2) * dim(shapes.y, 3) * dim(shapes.y, 4)" },
282
  { "name": "inChannels", "type": "u32", "value": "dim(shapes.x, 1)" },
@@ -292,10 +288,9 @@
292
  { "name": "outW", "type": "u32", "value": "dim(shapes.y, 4)" }
293
  ]
294
  },
295
- "cols_3": { "name": "cols", "buffer": "storage", "elementType": "$bScalar" },
296
- "params_13": {
297
  "name": "params",
298
- "buffer": "uniform",
299
  "struct": [
300
  { "name": "outCount", "type": "u32", "value": "dim(shapes.y, 2) * dim(shapes.y, 3)" },
301
  { "name": "inChannels", "type": "u32", "value": "dim(shapes.x, 1)" },
@@ -305,36 +300,16 @@
305
  { "name": "outW", "type": "u32", "value": "dim(shapes.y, 3)" }
306
  ]
307
  },
308
- "b_5": { "scratch": "cols3d", "name": "b", "buffer": "read-only-storage", "elementType": "$bScalar" },
309
- "b_6": { "scratch": "cols", "name": "b", "buffer": "read-only-storage", "elementType": "$bScalar" },
310
- "bias": { "arg": "B", "buffer": "read-only-storage", "elementType": "i32" }
311
  },
312
  "variants": [
313
  {
314
  "id": "dp4a_pointwise_1x1_padded_k_baseline",
315
  "priority": 21,
316
  "when": ["packedFeature", "not present.B", "not device.features.has(\"shader-f16\")", "rank4TensorOk", "tensorDtypes.x == \"uint8\"", "tensorDtypes.w == \"int8\"", "tensorDtypes.y == \"uint8\"", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.x, 1) % 4 != 0", "dim(shapes.x, 1) >= 4", "dim(shapes.w, 0) >= 8", "dim(shapes.y, 2) * dim(shapes.y, 3) >= 256", "dim(shapes.w, 2) == 1", "dim(shapes.w, 3) == 1", "strideH == 1", "strideW == 1", "padTop == 0 and padBottom == 0", "padLeft == 0 and padRight == 0", "dim(shapes.w, 1) == dim(shapes.x, 1)", "dim(shapes.y, 2) == dim(shapes.x, 2)", "dim(shapes.y, 3) == dim(shapes.x, 3)", "paddedMatricesStorageOk", "dp4a2dDispatchOk"],
317
- "derive": {
318
- "aZeroPerRow": "ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == dim(shapes.w, 0)",
319
- "aScalar": "\"i32\"",
320
- "bScalar": "\"u32\"",
321
- "yScalar": "\"u32\"",
322
- "aUnsigned": false,
323
- "bUnsigned": true,
324
- "yUnsigned": true,
325
- "hasAZero": true,
326
- "hasBZero": true,
327
- "aPacked": true,
328
- "bZeroPerColumn": false,
329
- "requant": "\"per_row\" if ranks.w_scale == 1 and dim(shapes.w_scale, 0) == dim(shapes.w, 0) else \"per_tensor\"",
330
- "scaleScalar": "\"f32\"",
331
- "qMin": 0,
332
- "qMax": 255,
333
- "tileM": 8,
334
- "tileN": 64,
335
- "ktWords": 32,
336
- "padB": true
337
- },
338
  "intermediates": [
339
  { "id": "w_padded", "dtype": "int32", "shape": "[dim(shapes.w, 0) * ceilDiv(dim(shapes.x, 1), 4) * 4]" },
340
  {
@@ -377,7 +352,7 @@
377
  "name": "QLinearConv.Dp4aPointwisePaddedK",
378
  "shader": "quant-dp4a-matmul.wgsl.jinja",
379
  "bindings": [
380
- "a_2",
381
  { "scratch": "x_padded", "name": "b", "buffer": "read-only-storage", "elementType": "u32" },
382
  "a_zero_point",
383
  "b_zero_point",
@@ -400,27 +375,7 @@
400
  "id": "dp4a_im2col_nchw_padded_k_baseline",
401
  "priority": 21,
402
  "when": ["packedFeature", "not present.B", "not device.features.has(\"shader-f16\")", "rank4TensorOk", "tensorDtypes.x == \"uint8\"", "tensorDtypes.w == \"int8\"", "tensorDtypes.y == \"uint8\"", "flexibleWeightQuantOk", "attrs.group == 1", "strideH >= 1", "strideW >= 1", "dilationH >= 1", "dilationW >= 1", "(dim(shapes.w, 2) > 1 or dim(shapes.w, 3) > 1)", "dim(shapes.w, 1) == dim(shapes.x, 1)", "dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) % 4 != 0", "dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) >= 64", "dim(shapes.w, 0) >= 8", "dim(shapes.y, 2) * dim(shapes.y, 3) >= 256", "numel(shapes.y) >= 8192", "output2dShapeOk", "paddedMatricesStorageOk", "paddedPreparationDispatchOk", "dp4a2dDispatchOk"],
403
- "derive": {
404
- "aZeroPerRow": "ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == dim(shapes.w, 0)",
405
- "aScalar": "\"i32\"",
406
- "bScalar": "\"u32\"",
407
- "yScalar": "\"u32\"",
408
- "aUnsigned": false,
409
- "bUnsigned": true,
410
- "yUnsigned": true,
411
- "hasAZero": true,
412
- "hasBZero": true,
413
- "aPacked": true,
414
- "bZeroPerColumn": false,
415
- "requant": "\"per_row\" if ranks.w_scale == 1 and dim(shapes.w_scale, 0) == dim(shapes.w, 0) else \"per_tensor\"",
416
- "scaleScalar": "\"f32\"",
417
- "qMin": 0,
418
- "qMax": 255,
419
- "tileM": 8,
420
- "tileN": 64,
421
- "ktWords": 32,
422
- "padB": false
423
- },
424
  "intermediates": [
425
  {
426
  "id": "w_padded",
@@ -513,7 +468,7 @@
513
  "name": "QLinearConv.Im2ColPaddedKDp4a",
514
  "shader": "quant-dp4a-matmul.wgsl.jinja",
515
  "bindings": [
516
- "a_2",
517
  { "scratch": "cols_padded", "name": "b", "buffer": "read-only-storage", "elementType": "u32" },
518
  "a_zero_point",
519
  "b_zero_point",
@@ -570,25 +525,8 @@
570
  "priority": 20,
571
  "when": ["packedFeature", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "(device.features.has(\"shader-f16\") or dim(shapes.x, 1) % 4 == 0)", "dim(shapes.w, 2) == 1", "dim(shapes.w, 3) == 1", "strideH == 1", "strideW == 1", "padTop == 0 and padBottom == 0", "padLeft == 0 and padRight == 0", "dim(shapes.w, 1) == dim(shapes.x, 1)", "dim(shapes.y, 2) == dim(shapes.x, 2)", "dim(shapes.y, 3) == dim(shapes.x, 3)", "dp4a2dDispatchOk"],
572
  "derive": {
573
- "aScalar": "\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\"",
574
- "bScalar": "\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\"",
575
  "aVec4": "(\"vec4<u32>\" if tensorDtypes.w == \"uint8\" else \"vec4<i32>\") if dim(shapes.x, 1) % 4 == 0 else (\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\")",
576
- "yScalar": "\"u32\" if tensorDtypes.y == \"uint8\" else \"i32\"",
577
- "aUnsigned": "tensorDtypes.w == \"uint8\"",
578
- "bUnsigned": "tensorDtypes.x == \"uint8\"",
579
- "yUnsigned": "tensorDtypes.y == \"uint8\"",
580
- "hasAZero": true,
581
- "hasBZero": true,
582
- "aPacked": "dim(shapes.x, 1) % 4 == 0",
583
- "aZeroPerRow": "ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == dim(shapes.w, 0)",
584
- "bZeroPerColumn": false,
585
- "requant": "\"per_row\" if ranks.w_scale == 1 and dim(shapes.w_scale, 0) == dim(shapes.w, 0) else \"per_tensor\"",
586
- "scaleScalar": "\"f32\"",
587
- "qMin": "0 if tensorDtypes.y == \"uint8\" else 0 - 128",
588
- "qMax": "255 if tensorDtypes.y == \"uint8\" else 127",
589
- "tileM": 8,
590
- "tileN": 64,
591
- "ktWords": 32
592
  },
593
  "passes": [
594
  {
@@ -596,16 +534,16 @@
596
  "name": "QLinearConv.Dp4aPointwiseFused",
597
  "shader": "quant-dp4a-matmul.wgsl.jinja",
598
  "bindings": [
599
- "a_3",
600
  { "arg": "x", "name": "b", "elementType": "$bScalar" },
601
- "a_zero_point_2",
602
- "b_zero_point_2",
603
  "a_scale",
604
  "b_scale",
605
  "y_scale",
606
- "y_zero_point_2",
607
- "y_2",
608
- "params_5"
609
  ],
610
  "dispatch": {
611
  "x": "ceil(dim(shapes.y, 2) * dim(shapes.y, 3) / 64)",
@@ -620,17 +558,9 @@
620
  "priority": 10,
621
  "when": ["not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "groupChannelsOk", "output2dShapeOk", "dim(shapes.y, 3) >= 4", "dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) >= 4", "numel(shapes.y) >= 32"],
622
  "derive": {
623
- "xScalar": "\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\"",
624
- "wScalar": "\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\"",
625
- "yScalar": "\"u32\" if tensorDtypes.y == \"uint8\" else \"i32\"",
626
- "xUnsigned": "tensorDtypes.x == \"uint8\"",
627
- "wUnsigned": "tensorDtypes.w == \"uint8\"",
628
- "yUnsigned": "tensorDtypes.y == \"uint8\"",
629
  "wScalePerChannel": "ranks.w_scale == 1 and dim(shapes.w_scale, 0) == dim(shapes.w, 0)",
630
  "wZeroPerChannel": "ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == dim(shapes.w, 0)",
631
- "ocTile": "4 if dim(shapes.w, 0) >= attrs.group * 4 and dim(shapes.w, 0) % (attrs.group * 4) == 0 else 1",
632
- "qMin": "0 if tensorDtypes.y == \"uint8\" else 0 - 128",
633
- "qMax": "255 if tensorDtypes.y == \"uint8\" else 127"
634
  },
635
  "passes": [
636
  {
@@ -638,15 +568,15 @@
638
  "name": "QLinearConv.FeatureIndependentX4",
639
  "shader": "qlinear-conv-nchw-x4.wgsl.jinja",
640
  "bindings": [
641
- "x_2",
642
  "x_scale",
643
- "x_zero_point_2",
644
- "w_2",
645
  "w_scale",
646
- "w_zero_point_2",
647
  "y_scale",
648
- "y_zero_point_2",
649
- "y_2",
650
  {
651
  "name": "params",
652
  "struct": [
@@ -682,19 +612,7 @@
682
  {
683
  "id": "ncdhw3d_per_channel_quantization",
684
  "when": ["not present.B", "rank5TensorOk and dim(shapes.y, 2) == expectedOutputDepth and dim(shapes.y, 3) == expectedOutputHeight and dim(shapes.y, 4) == expectedOutputWidth", "not scalarWeightQuantOk and ranks.w_scale == 1 and dim(shapes.w_scale, 0) == dim(shapes.w, 0) and ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == dim(shapes.w, 0)", "groupChannelsOk"],
685
- "derive": {
686
- "xScalar": "\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\"",
687
- "wScalar": "\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\"",
688
- "yScalar": "\"u32\" if tensorDtypes.y == \"uint8\" else \"i32\"",
689
- "xUnsigned": "tensorDtypes.x == \"uint8\"",
690
- "wUnsigned": "tensorDtypes.w == \"uint8\"",
691
- "yUnsigned": "tensorDtypes.y == \"uint8\"",
692
- "hasBias": false,
693
- "wScalePerChannel": true,
694
- "wZeroPerChannel": true,
695
- "qMin": "0 if tensorDtypes.y == \"uint8\" else 0 - 128",
696
- "qMax": "255 if tensorDtypes.y == \"uint8\" else 127"
697
- },
698
  "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }],
699
  "passes": [
700
  {
@@ -702,7 +620,7 @@
702
  "name": "QLinearConv.AccumulatePerChannel3d",
703
  "shader": "conv-int-accumulate-spatial.wgsl.jinja",
704
  "derive": { "spatialRankSpec": 3 },
705
- "bindings": ["x_2", "x_zero_point_2", "w_2", "w_zero_point_2", "y_3", "params_7"],
706
  "dispatch": {
707
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
708
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
@@ -713,7 +631,7 @@
713
  "id": "requantize",
714
  "name": "QLinearConv.Requantize",
715
  "shader": "qlinear-conv-requantize.wgsl.jinja",
716
- "bindings": ["accum", "x_scale", "w_scale", "y_scale", "y_zero_point_2", "y_2", "params_10"],
717
  "dispatch": {
718
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
719
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
@@ -725,17 +643,7 @@
725
  {
726
  "id": "ncw1d_scalar_quantization",
727
  "when": ["not present.B", "rank3TensorOk and dim(shapes.y, 2) == expectedOutputWidth", "scalarWeightQuantOk", "groupChannelsOk"],
728
- "derive": {
729
- "xScalar": "\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\"",
730
- "wScalar": "\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\"",
731
- "yScalar": "\"u32\" if tensorDtypes.y == \"uint8\" else \"i32\"",
732
- "xUnsigned": "tensorDtypes.x == \"uint8\"",
733
- "wUnsigned": "tensorDtypes.w == \"uint8\"",
734
- "yUnsigned": "tensorDtypes.y == \"uint8\"",
735
- "hasBias": false,
736
- "qMin": "0 if tensorDtypes.y == \"uint8\" else 0 - 128",
737
- "qMax": "255 if tensorDtypes.y == \"uint8\" else 127"
738
- },
739
  "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }],
740
  "passes": [
741
  {
@@ -743,7 +651,7 @@
743
  "name": "QLinearConv.Accumulate1d",
744
  "shader": "conv-int-accumulate-spatial.wgsl.jinja",
745
  "derive": { "spatialRankSpec": 2 },
746
- "bindings": ["x_2", "x_zero_point_2", "w_2", "w_zero_point_2", "y_3", "params_8"],
747
  "dispatch": {
748
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
749
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
@@ -754,7 +662,7 @@
754
  "id": "requantize",
755
  "name": "QLinearConv.Requantize",
756
  "shader": "qlinear-conv-requantize.wgsl.jinja",
757
- "bindings": ["accum", "x_scale", "w_scale", "y_scale", "y_zero_point_2", "y_2", "params_11"],
758
  "dispatch": {
759
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
760
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
@@ -766,17 +674,7 @@
766
  {
767
  "id": "ncdhw3d_scalar_quantization",
768
  "when": ["not present.B", "rank5TensorOk and dim(shapes.y, 2) == expectedOutputDepth and dim(shapes.y, 3) == expectedOutputHeight and dim(shapes.y, 4) == expectedOutputWidth", "scalarWeightQuantOk", "groupChannelsOk"],
769
- "derive": {
770
- "xScalar": "\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\"",
771
- "wScalar": "\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\"",
772
- "yScalar": "\"u32\" if tensorDtypes.y == \"uint8\" else \"i32\"",
773
- "xUnsigned": "tensorDtypes.x == \"uint8\"",
774
- "wUnsigned": "tensorDtypes.w == \"uint8\"",
775
- "yUnsigned": "tensorDtypes.y == \"uint8\"",
776
- "hasBias": false,
777
- "qMin": "0 if tensorDtypes.y == \"uint8\" else 0 - 128",
778
- "qMax": "255 if tensorDtypes.y == \"uint8\" else 127"
779
- },
780
  "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }],
781
  "passes": [
782
  {
@@ -784,7 +682,7 @@
784
  "name": "QLinearConv.Accumulate3d",
785
  "shader": "conv-int-accumulate-spatial.wgsl.jinja",
786
  "derive": { "spatialRankSpec": 3 },
787
- "bindings": ["x_2", "x_zero_point_2", "w_2", "w_zero_point_2", "y_3", "params_7"],
788
  "dispatch": {
789
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
790
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
@@ -795,7 +693,7 @@
795
  "id": "requantize",
796
  "name": "QLinearConv.Requantize",
797
  "shader": "qlinear-conv-requantize.wgsl.jinja",
798
- "bindings": ["accum", "x_scale", "w_scale", "y_scale", "y_zero_point_2", "y_2", "params_11"],
799
  "dispatch": {
800
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
801
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
@@ -807,17 +705,7 @@
807
  {
808
  "id": "nchw2d_scalar_quantization",
809
  "when": ["not present.B", "rank4TensorOk and output2dShapeOk", "scalarWeightQuantOk", "groupChannelsOk"],
810
- "derive": {
811
- "xScalar": "\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\"",
812
- "wScalar": "\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\"",
813
- "yScalar": "\"u32\" if tensorDtypes.y == \"uint8\" else \"i32\"",
814
- "xUnsigned": "tensorDtypes.x == \"uint8\"",
815
- "wUnsigned": "tensorDtypes.w == \"uint8\"",
816
- "yUnsigned": "tensorDtypes.y == \"uint8\"",
817
- "hasBias": false,
818
- "qMin": "0 if tensorDtypes.y == \"uint8\" else 0 - 128",
819
- "qMax": "255 if tensorDtypes.y == \"uint8\" else 127"
820
- },
821
  "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }],
822
  "passes": [
823
  {
@@ -825,7 +713,7 @@
825
  "name": "QLinearConv.Accumulate",
826
  "shader": "conv-int-accumulate-spatial.wgsl.jinja",
827
  "derive": { "spatialRankSpec": 2 },
828
- "bindings": ["x_2", "x_zero_point_2", "w_2", "w_zero_point_2", "y_3", "params_9"],
829
  "dispatch": {
830
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
831
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
@@ -836,7 +724,7 @@
836
  "id": "requantize",
837
  "name": "QLinearConv.Requantize",
838
  "shader": "qlinear-conv-requantize.wgsl.jinja",
839
- "bindings": ["accum", "x_scale", "w_scale", "y_scale", "y_zero_point_2", "y_2", "params_11"],
840
  "dispatch": {
841
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
842
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
@@ -848,17 +736,7 @@
848
  {
849
  "id": "ncw1d_scalar_quantization_bias",
850
  "when": ["present.B and ranks.B == 1 and dim(shapes.B, 0) == dim(shapes.w, 0)", "rank3TensorOk and dim(shapes.y, 2) == expectedOutputWidth", "scalarWeightQuantOk", "groupChannelsOk"],
851
- "derive": {
852
- "xScalar": "\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\"",
853
- "wScalar": "\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\"",
854
- "yScalar": "\"u32\" if tensorDtypes.y == \"uint8\" else \"i32\"",
855
- "xUnsigned": "tensorDtypes.x == \"uint8\"",
856
- "wUnsigned": "tensorDtypes.w == \"uint8\"",
857
- "yUnsigned": "tensorDtypes.y == \"uint8\"",
858
- "hasBias": true,
859
- "qMin": "0 if tensorDtypes.y == \"uint8\" else 0 - 128",
860
- "qMax": "255 if tensorDtypes.y == \"uint8\" else 127"
861
- },
862
  "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }],
863
  "passes": [
864
  {
@@ -866,7 +744,7 @@
866
  "name": "QLinearConv.Accumulate1d",
867
  "shader": "conv-int-accumulate-spatial.wgsl.jinja",
868
  "derive": { "spatialRankSpec": 2 },
869
- "bindings": ["x_2", "x_zero_point_2", "w_2", "w_zero_point_2", "y_3", "params_8"],
870
  "dispatch": {
871
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
872
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
@@ -877,7 +755,7 @@
877
  "id": "requantize",
878
  "name": "QLinearConv.Requantize",
879
  "shader": "qlinear-conv-requantize.wgsl.jinja",
880
- "bindings": ["accum", "x_scale", "w_scale", "y_scale", "y_zero_point_2", "B", "y_2", "params_10"],
881
  "dispatch": {
882
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
883
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
@@ -889,17 +767,7 @@
889
  {
890
  "id": "ncdhw3d_scalar_quantization_bias",
891
  "when": ["present.B and ranks.B == 1 and dim(shapes.B, 0) == dim(shapes.w, 0)", "rank5TensorOk and dim(shapes.y, 2) == expectedOutputDepth and dim(shapes.y, 3) == expectedOutputHeight and dim(shapes.y, 4) == expectedOutputWidth", "scalarWeightQuantOk", "groupChannelsOk"],
892
- "derive": {
893
- "xScalar": "\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\"",
894
- "wScalar": "\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\"",
895
- "yScalar": "\"u32\" if tensorDtypes.y == \"uint8\" else \"i32\"",
896
- "xUnsigned": "tensorDtypes.x == \"uint8\"",
897
- "wUnsigned": "tensorDtypes.w == \"uint8\"",
898
- "yUnsigned": "tensorDtypes.y == \"uint8\"",
899
- "hasBias": true,
900
- "qMin": "0 if tensorDtypes.y == \"uint8\" else 0 - 128",
901
- "qMax": "255 if tensorDtypes.y == \"uint8\" else 127"
902
- },
903
  "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }],
904
  "passes": [
905
  {
@@ -907,7 +775,7 @@
907
  "name": "QLinearConv.Accumulate3d",
908
  "shader": "conv-int-accumulate-spatial.wgsl.jinja",
909
  "derive": { "spatialRankSpec": 3 },
910
- "bindings": ["x_2", "x_zero_point_2", "w_2", "w_zero_point_2", "y_3", "params_7"],
911
  "dispatch": {
912
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
913
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
@@ -918,7 +786,7 @@
918
  "id": "requantize",
919
  "name": "QLinearConv.Requantize",
920
  "shader": "qlinear-conv-requantize.wgsl.jinja",
921
- "bindings": ["accum", "x_scale", "w_scale", "y_scale", "y_zero_point_2", "B", "y_2", "params_10"],
922
  "dispatch": {
923
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
924
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
@@ -930,19 +798,7 @@
930
  {
931
  "id": "ncw1d_per_channel_quantization",
932
  "when": ["not present.B", "rank3TensorOk and dim(shapes.y, 2) == expectedOutputWidth", "not scalarWeightQuantOk and ranks.w_scale == 1 and dim(shapes.w_scale, 0) == dim(shapes.w, 0) and ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == dim(shapes.w, 0)", "groupChannelsOk"],
933
- "derive": {
934
- "xScalar": "\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\"",
935
- "wScalar": "\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\"",
936
- "yScalar": "\"u32\" if tensorDtypes.y == \"uint8\" else \"i32\"",
937
- "xUnsigned": "tensorDtypes.x == \"uint8\"",
938
- "wUnsigned": "tensorDtypes.w == \"uint8\"",
939
- "yUnsigned": "tensorDtypes.y == \"uint8\"",
940
- "hasBias": false,
941
- "wScalePerChannel": true,
942
- "wZeroPerChannel": true,
943
- "qMin": "0 if tensorDtypes.y == \"uint8\" else 0 - 128",
944
- "qMax": "255 if tensorDtypes.y == \"uint8\" else 127"
945
- },
946
  "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }],
947
  "passes": [
948
  {
@@ -950,7 +806,7 @@
950
  "name": "QLinearConv.AccumulatePerChannel1d",
951
  "shader": "conv-int-accumulate-spatial.wgsl.jinja",
952
  "derive": { "spatialRankSpec": 2 },
953
- "bindings": ["x_2", "x_zero_point_2", "w_2", "w_zero_point_2", "y_3", "params_8"],
954
  "dispatch": {
955
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
956
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
@@ -961,7 +817,7 @@
961
  "id": "requantize",
962
  "name": "QLinearConv.Requantize",
963
  "shader": "qlinear-conv-requantize.wgsl.jinja",
964
- "bindings": ["accum", "x_scale", "w_scale", "y_scale", "y_zero_point_2", "y_2", "params_10"],
965
  "dispatch": {
966
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
967
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
@@ -975,25 +831,8 @@
975
  "priority": 20,
976
  "when": ["packedFeature", "not present.B", "rank5TensorOk", "attrs.group == 1", "dim(shapes.w, 1) == dim(shapes.x, 1)", "dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) * dim(shapes.w, 4) % 4 == 0", "dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) * dim(shapes.w, 4) >= 32", "dim(shapes.w, 0) >= 8", "dim(shapes.y, 2) * dim(shapes.y, 3) * dim(shapes.y, 4) >= 256", "flexibleWeightQuantOk", "dim(shapes.y, 2) == expectedOutputDepth", "dim(shapes.y, 3) == expectedOutputHeight", "dim(shapes.y, 4) == expectedOutputWidth", "im2colStorageOk", "dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) * dim(shapes.w, 4) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "ceilDiv(dim(shapes.y, 2) * dim(shapes.y, 3) * dim(shapes.y, 4), tunables.WORKGROUP_SIZE) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "ceilDiv(dim(shapes.y, 2) * dim(shapes.y, 3) * dim(shapes.y, 4), 64) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "ceilDiv(dim(shapes.w, 0), 8) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "dim(shapes.x, 0) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)"],
977
  "derive": {
978
- "aScalar": "\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\"",
979
- "bScalar": "\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\"",
980
  "aVec4": "\"vec4<u32>\" if tensorDtypes.w == \"uint8\" else \"vec4<i32>\"",
981
- "yScalar": "\"u32\" if tensorDtypes.y == \"uint8\" else \"i32\"",
982
- "aUnsigned": "tensorDtypes.w == \"uint8\"",
983
- "bUnsigned": "tensorDtypes.x == \"uint8\"",
984
- "yUnsigned": "tensorDtypes.y == \"uint8\"",
985
- "hasAZero": true,
986
- "hasBZero": true,
987
  "aPacked": true,
988
- "aZeroPerRow": "ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == dim(shapes.w, 0)",
989
- "bZeroPerColumn": false,
990
- "requant": "\"per_row\" if ranks.w_scale == 1 and dim(shapes.w_scale, 0) == dim(shapes.w, 0) else \"per_tensor\"",
991
- "scaleScalar": "\"f32\"",
992
- "qMin": "0 if tensorDtypes.y == \"uint8\" else 0 - 128",
993
- "qMax": "255 if tensorDtypes.y == \"uint8\" else 127",
994
- "tileM": 8,
995
- "tileN": 64,
996
- "ktWords": 32,
997
  "hasBias": false
998
  },
999
  "intermediates": [
@@ -1023,7 +862,7 @@
1023
  "padTopSpec": "effectivePadTop",
1024
  "padLeftSpec": "effectivePadLeft"
1025
  },
1026
- "bindings": ["x_3", "x_zero_point_3", "cols_2", "params_12"],
1027
  "dispatch": {
1028
  "x": "ceil(dim(shapes.y, 2) * dim(shapes.y, 3) * dim(shapes.y, 4) / tunables.WORKGROUP_SIZE)",
1029
  "y": "dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) * dim(shapes.w, 4)",
@@ -1034,7 +873,7 @@
1034
  "id": "main",
1035
  "name": "QLinearConv.Im2Col3dDp4a",
1036
  "shader": "quant-dp4a-matmul.wgsl.jinja",
1037
- "bindings": ["a_3", "b_5", "a_zero_point_2", "b_zero_point_2", "a_scale", "b_scale", "y_scale", "y_zero_point_2", "y_2", "params_5"],
1038
  "dispatch": {
1039
  "x": "ceil(dim(shapes.y, 2) * dim(shapes.y, 3) * dim(shapes.y, 4) / 64)",
1040
  "y": "ceil(dim(shapes.w, 0) / 8)",
@@ -1048,25 +887,8 @@
1048
  "priority": 25,
1049
  "when": ["packedFeature", "present.B", "ranks.B == 1", "dim(shapes.B, 0) == dim(shapes.w, 0)", "rank4TensorOk", "attrs.group == 1", "dim(shapes.w, 1) == dim(shapes.x, 1)", "flexibleWeightQuantOk"],
1050
  "derive": {
1051
- "aScalar": "\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\"",
1052
- "bScalar": "\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\"",
1053
  "aVec4": "(\"vec4<u32>\" if tensorDtypes.w == \"uint8\" else \"vec4<i32>\") if dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) % 4 == 0 else (\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\")",
1054
- "yScalar": "\"u32\" if tensorDtypes.y == \"uint8\" else \"i32\"",
1055
- "aUnsigned": "tensorDtypes.w == \"uint8\"",
1056
- "bUnsigned": "tensorDtypes.x == \"uint8\"",
1057
- "yUnsigned": "tensorDtypes.y == \"uint8\"",
1058
- "hasAZero": true,
1059
- "hasBZero": true,
1060
  "aPacked": "dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) % 4 == 0",
1061
- "aZeroPerRow": "ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == dim(shapes.w, 0)",
1062
- "bZeroPerColumn": false,
1063
- "requant": "\"per_row\" if ranks.w_scale == 1 and dim(shapes.w_scale, 0) == dim(shapes.w, 0) else \"per_tensor\"",
1064
- "scaleScalar": "\"f32\"",
1065
- "qMin": "0 if tensorDtypes.y == \"uint8\" else 0 - 128",
1066
- "qMax": "255 if tensorDtypes.y == \"uint8\" else 127",
1067
- "tileM": 8,
1068
- "tileN": 64,
1069
- "ktWords": 32,
1070
  "hasBias": true
1071
  },
1072
  "intermediates": [
@@ -1092,7 +914,7 @@
1092
  "padTopSpec": "effectivePadTop",
1093
  "padLeftSpec": "effectivePadLeft"
1094
  },
1095
- "bindings": ["x_3", "x_zero_point_3", "cols_3", "params_13"],
1096
  "dispatch": {
1097
  "x": "ceil(dim(shapes.y, 2) * dim(shapes.y, 3) / tunables.WORKGROUP_SIZE)",
1098
  "y": "dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3)",
@@ -1103,7 +925,7 @@
1103
  "id": "main",
1104
  "name": "QLinearConv.Im2ColDp4aBias",
1105
  "shader": "quant-dp4a-matmul.wgsl.jinja",
1106
- "bindings": ["a_3", "b_6", "a_zero_point_2", "b_zero_point_2", "a_scale", "b_scale", "y_scale", "y_zero_point_2", "bias", "y_2", "params_5"],
1107
  "dispatch": {
1108
  "x": "ceil(dim(shapes.y, 2) * dim(shapes.y, 3) / 64)",
1109
  "y": "ceil(dim(shapes.w, 0) / 8)",
@@ -1117,25 +939,8 @@
1117
  "priority": 20,
1118
  "when": ["packedFeature", "not present.B", "flexibleWeightQuantOk", "rank4TensorOk", "attrs.group == 1", "(device.features.has(\"shader-f16\") or dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) % 4 == 0)", "(dim(shapes.w, 2) > 1 or dim(shapes.w, 3) > 1)", "dim(shapes.w, 1) == dim(shapes.x, 1)", "dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) >= 64", "numel(shapes.y) >= 8192", "dp4a2dDispatchOk"],
1119
  "derive": {
1120
- "aScalar": "\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\"",
1121
- "bScalar": "\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\"",
1122
  "aVec4": "(\"vec4<u32>\" if tensorDtypes.w == \"uint8\" else \"vec4<i32>\") if dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) % 4 == 0 else (\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\")",
1123
- "yScalar": "\"u32\" if tensorDtypes.y == \"uint8\" else \"i32\"",
1124
- "aUnsigned": "tensorDtypes.w == \"uint8\"",
1125
- "bUnsigned": "tensorDtypes.x == \"uint8\"",
1126
- "yUnsigned": "tensorDtypes.y == \"uint8\"",
1127
- "hasAZero": true,
1128
- "hasBZero": true,
1129
- "aPacked": "dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) % 4 == 0",
1130
- "aZeroPerRow": "ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == dim(shapes.w, 0)",
1131
- "bZeroPerColumn": false,
1132
- "requant": "\"per_row\" if ranks.w_scale == 1 and dim(shapes.w_scale, 0) == dim(shapes.w, 0) else \"per_tensor\"",
1133
- "scaleScalar": "\"f32\"",
1134
- "qMin": "0 if tensorDtypes.y == \"uint8\" else 0 - 128",
1135
- "qMax": "255 if tensorDtypes.y == \"uint8\" else 127",
1136
- "tileM": 8,
1137
- "tileN": 64,
1138
- "ktWords": 32
1139
  },
1140
  "intermediates": [
1141
  {
@@ -1160,7 +965,7 @@
1160
  "padTopSpec": "effectivePadTop",
1161
  "padLeftSpec": "effectivePadLeft"
1162
  },
1163
- "bindings": ["x_3", "x_zero_point_3", "cols_3", "params_13"],
1164
  "dispatch": {
1165
  "x": "ceil(dim(shapes.y, 2) * dim(shapes.y, 3) / tunables.WORKGROUP_SIZE)",
1166
  "y": "dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3)",
@@ -1171,7 +976,7 @@
1171
  "id": "main",
1172
  "name": "QLinearConv.Im2ColDp4a",
1173
  "shader": "quant-dp4a-matmul.wgsl.jinja",
1174
- "bindings": ["a_3", "b_6", "a_zero_point_2", "b_zero_point_2", "a_scale", "b_scale", "y_scale", "y_zero_point_2", "y_2", "params_5"],
1175
  "dispatch": {
1176
  "x": "ceil(dim(shapes.y, 2) * dim(shapes.y, 3) / 64)",
1177
  "y": "ceil(dim(shapes.w, 0) / 8)",
@@ -1184,19 +989,7 @@
1184
  "id": "nchw2d_per_channel_quantization",
1185
  "priority": 5,
1186
  "when": ["not present.B", "rank4TensorOk", "ranks.w_scale == 1", "dim(shapes.w_scale, 0) == dim(shapes.w, 0)", "ranks.w_zero_point == 1", "dim(shapes.w_zero_point, 0) == dim(shapes.w, 0)", "groupChannelsOk", "output2dShapeOk"],
1187
- "derive": {
1188
- "xScalar": "\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\"",
1189
- "wScalar": "\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\"",
1190
- "yScalar": "\"u32\" if tensorDtypes.y == \"uint8\" else \"i32\"",
1191
- "xUnsigned": "tensorDtypes.x == \"uint8\"",
1192
- "wUnsigned": "tensorDtypes.w == \"uint8\"",
1193
- "yUnsigned": "tensorDtypes.y == \"uint8\"",
1194
- "hasBias": false,
1195
- "wScalePerChannel": true,
1196
- "wZeroPerChannel": true,
1197
- "qMin": "0 if tensorDtypes.y == \"uint8\" else 0 - 128",
1198
- "qMax": "255 if tensorDtypes.y == \"uint8\" else 127"
1199
- },
1200
  "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }],
1201
  "passes": [
1202
  {
@@ -1204,7 +997,7 @@
1204
  "name": "QLinearConv.AccumulatePerChannel",
1205
  "shader": "conv-int-accumulate-spatial.wgsl.jinja",
1206
  "derive": { "spatialRankSpec": 2 },
1207
- "bindings": ["x_2", "x_zero_point_2", "w_2", "w_zero_point_2", "y_3", "params_9"],
1208
  "dispatch": {
1209
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
1210
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
@@ -1215,7 +1008,7 @@
1215
  "id": "requantize",
1216
  "name": "QLinearConv.RequantizePerChannel",
1217
  "shader": "qlinear-conv-requantize.wgsl.jinja",
1218
- "bindings": ["accum", "x_scale", "w_scale", "y_scale", "y_zero_point_2", "y_2", "params_10"],
1219
  "dispatch": {
1220
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
1221
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
@@ -1227,19 +1020,7 @@
1227
  {
1228
  "id": "nchw2d_per_channel_quantization_bias",
1229
  "when": ["present.B", "ranks.B == 1", "dim(shapes.B, 0) == dim(shapes.w, 0)", "rank4TensorOk", "ranks.w_scale == 1", "dim(shapes.w_scale, 0) == dim(shapes.w, 0)", "ranks.w_zero_point == 1", "dim(shapes.w_zero_point, 0) == dim(shapes.w, 0)", "groupChannelsOk", "output2dShapeOk"],
1230
- "derive": {
1231
- "xScalar": "\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\"",
1232
- "wScalar": "\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\"",
1233
- "yScalar": "\"u32\" if tensorDtypes.y == \"uint8\" else \"i32\"",
1234
- "xUnsigned": "tensorDtypes.x == \"uint8\"",
1235
- "wUnsigned": "tensorDtypes.w == \"uint8\"",
1236
- "yUnsigned": "tensorDtypes.y == \"uint8\"",
1237
- "hasBias": true,
1238
- "wScalePerChannel": true,
1239
- "wZeroPerChannel": true,
1240
- "qMin": "0 if tensorDtypes.y == \"uint8\" else 0 - 128",
1241
- "qMax": "255 if tensorDtypes.y == \"uint8\" else 127"
1242
- },
1243
  "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }],
1244
  "passes": [
1245
  {
@@ -1247,7 +1028,7 @@
1247
  "name": "QLinearConv.AccumulatePerChannel",
1248
  "shader": "conv-int-accumulate-spatial.wgsl.jinja",
1249
  "derive": { "spatialRankSpec": 2 },
1250
- "bindings": ["x_2", "x_zero_point_2", "w_2", "w_zero_point_2", "y_3", "params_9"],
1251
  "dispatch": {
1252
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
1253
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
@@ -1258,7 +1039,7 @@
1258
  "id": "requantize",
1259
  "name": "QLinearConv.RequantizePerChannelBias",
1260
  "shader": "qlinear-conv-requantize.wgsl.jinja",
1261
- "bindings": ["accum", "x_scale", "w_scale", "y_scale", "y_zero_point_2", "B", "y_2", "params_10"],
1262
  "dispatch": {
1263
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
1264
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
@@ -1270,17 +1051,7 @@
1270
  {
1271
  "id": "nchw2d_scalar_quantization_bias",
1272
  "when": ["present.B", "ranks.B == 1", "dim(shapes.B, 0) == dim(shapes.w, 0)", "rank4TensorOk", "scalarWeightQuantOk", "groupChannelsOk", "output2dShapeOk"],
1273
- "derive": {
1274
- "xScalar": "\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\"",
1275
- "wScalar": "\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\"",
1276
- "yScalar": "\"u32\" if tensorDtypes.y == \"uint8\" else \"i32\"",
1277
- "xUnsigned": "tensorDtypes.x == \"uint8\"",
1278
- "wUnsigned": "tensorDtypes.w == \"uint8\"",
1279
- "yUnsigned": "tensorDtypes.y == \"uint8\"",
1280
- "hasBias": true,
1281
- "qMin": "0 if tensorDtypes.y == \"uint8\" else 0 - 128",
1282
- "qMax": "255 if tensorDtypes.y == \"uint8\" else 127"
1283
- },
1284
  "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }],
1285
  "passes": [
1286
  {
@@ -1288,7 +1059,7 @@
1288
  "name": "QLinearConv.Accumulate",
1289
  "shader": "conv-int-accumulate-spatial.wgsl.jinja",
1290
  "derive": { "spatialRankSpec": 2 },
1291
- "bindings": ["x_2", "x_zero_point_2", "w_2", "w_zero_point_2", "y_3", "params_9"],
1292
  "dispatch": {
1293
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
1294
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
@@ -1299,7 +1070,7 @@
1299
  "id": "requantize",
1300
  "name": "QLinearConv.RequantizeBias",
1301
  "shader": "qlinear-conv-requantize.wgsl.jinja",
1302
- "bindings": ["accum", "x_scale", "w_scale", "y_scale", "y_zero_point_2", "B", "y_2", "params_10"],
1303
  "dispatch": {
1304
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
1305
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
 
90
  "explicitPadsOk": "attrs.auto_pad == \"NOTSET\" or not has(attrs, \"pads\")",
91
  "spatialAttributesOk": "spatialRank >= 1 and spatialRank <= 3 and spatialAttributeLengthsOk and kernelShapeMatchesWeights and kernelExtentsOk and stridesValuesOk and dilationsValuesOk and padsValuesOk and explicitPadsOk",
92
  "packedFeature": "device.wgslLanguageFeatures.has(\"packed_4x8_integer_dot_product\")",
 
 
93
  "rank4TensorOk": "spatialAttributesOk and (ranks.x == 4 and ranks.w == 4 and ranks.y == 4)",
 
94
  "outputBatchChannelsOk": "dim(shapes.y, 0) == dim(shapes.x, 0) and dim(shapes.y, 1) == dim(shapes.w, 0)",
95
  "groupChannelsOk": "attrs.group >= 1 and dim(shapes.w, 1) * attrs.group == dim(shapes.x, 1) and dim(shapes.w, 0) % attrs.group == 0",
96
  "output2dShapeOk": "dim(shapes.y, 2) == expectedOutputHeight and dim(shapes.y, 3) == expectedOutputWidth",
97
+ "aScalar": "\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\"",
98
+ "bScalar": "\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\"",
99
+ "wScalar": "\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\"",
100
+ "xScalar": "\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\"",
101
+ "aUnsigned": "tensorDtypes.w == \"uint8\"",
102
+ "bUnsigned": "tensorDtypes.x == \"uint8\"",
103
+ "wUnsigned": "tensorDtypes.w == \"uint8\"",
104
+ "xUnsigned": "tensorDtypes.x == \"uint8\"",
105
+ "bZeroPerColumn": false,
106
+ "xyQuantContract": "(ranks.x_scale == 0 or (ranks.x_scale == 1 and dim(shapes.x_scale, 0) == 1)) and (ranks.y_scale == 0 or (ranks.y_scale == 1 and dim(shapes.y_scale, 0) == 1)) and (ranks.x_zero_point == 0 or (ranks.x_zero_point == 1 and dim(shapes.x_zero_point, 0) == 1)) and (ranks.y_zero_point == 0 or (ranks.y_zero_point == 1 and dim(shapes.y_zero_point, 0) == 1))",
107
+ "rank3TensorOk": "spatialAttributesOk and ranks.x == 3 and ranks.w == 3 and ranks.y == 3",
108
+ "rank5TensorOk": "spatialAttributesOk and ranks.x == 5 and ranks.w == 5 and ranks.y == 5",
109
  "scalarWeightQuantOk": "(ranks.w_scale == 0 or (ranks.w_scale == 1 and dim(shapes.w_scale, 0) == 1)) and (ranks.w_zero_point == 0 or (ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == 1))",
110
+ "yScalar": "\"u32\" if tensorDtypes.y == \"uint8\" else \"i32\"",
111
+ "yUnsigned": "tensorDtypes.y == \"uint8\"",
112
+ "qMin": "0 if tensorDtypes.y == \"uint8\" else 0 - 128",
113
+ "qMax": "255 if tensorDtypes.y == \"uint8\" else 127",
114
  "flexibleWeightQuantOk": "(((ranks.w_scale == 0 or (ranks.w_scale == 1 and dim(shapes.w_scale, 0) == 1)) and (ranks.w_zero_point == 0 or (ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == 1))) or (ranks.w_scale == 1 and dim(shapes.w_scale, 0) == dim(shapes.w, 0) and ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == dim(shapes.w, 0)))",
115
  "dp4a2dDispatchOk": "ceilDiv(dim(shapes.y, 2) * dim(shapes.y, 3), 64) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(dim(shapes.w, 0), 8) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and dim(shapes.y, 0) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)",
116
  "convM": "dim(shapes.w, 0)",
 
125
  "paddedPreparationDispatchOk": "ceilDiv(inner(shapes.y, 1), tunables.WORKGROUP_SIZE) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(dim(shapes.w, 1) * inner(shapes.w, 1), 4) * 4 <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(dim(shapes.w, 0) * ceilDiv(dim(shapes.w, 1) * inner(shapes.w, 1), 4) * 4, tunables.WORKGROUP_SIZE) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)",
126
  "direct3x3SharedBytes": "4 * ((tunables.DP4A_DIRECT_IC_TILE / 4) * (tunables.DP4A_DIRECT_SPATIAL_TILE + 2) * (tunables.DP4A_DIRECT_SPATIAL_TILE + 2) + tunables.DP4A_DIRECT_OC_TILE * (tunables.DP4A_DIRECT_IC_TILE / 4) * 9 + tunables.DP4A_DIRECT_OC_TILE)",
127
  "direct3x3DispatchOk": "ceilDiv(dim(shapes.y, 3), tunables.DP4A_DIRECT_SPATIAL_TILE) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(dim(shapes.y, 2), tunables.DP4A_DIRECT_SPATIAL_TILE) * ceilDiv(dim(shapes.w, 0), tunables.DP4A_DIRECT_OC_TILE) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and dim(shapes.y, 0) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)",
128
+ "direct3x3DeviceOk": "tunables.DP4A_DIRECT_IC_TILE % 4 == 0 and tunables.DP4A_DIRECT_SPATIAL_TILE * tunables.DP4A_DIRECT_SPATIAL_TILE <= device.limits.maxComputeInvocationsPerWorkgroup and tunables.DP4A_DIRECT_SPATIAL_TILE <= device.limits.maxComputeWorkgroupSizeX and tunables.DP4A_DIRECT_SPATIAL_TILE <= device.limits.maxComputeWorkgroupSizeY and direct3x3SharedBytes <= device.limits.maxComputeWorkgroupStorageSize and direct3x3DispatchOk",
129
+ "aZeroPerRow": "ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == dim(shapes.w, 0)",
130
+ "requant": "\"per_row\" if ranks.w_scale == 1 and dim(shapes.w_scale, 0) == dim(shapes.w, 0) else \"per_tensor\"",
131
+ "tileM": 8,
132
+ "tileN": 64,
133
+ "ktWords": 32
134
  },
135
  "when": ["outputBatchChannelsOk", "xyQuantContract"],
136
  "bindings": {
137
+ "a": { "arg": "w", "elementType": "i32" },
138
+ "a_zero_point": { "arg": "w_zero_point", "elementType": "i32" },
139
+ "b_zero_point": { "arg": "x_zero_point", "elementType": "u32", "length": 1 },
140
+ "a_padded": { "scratch": "w_padded", "elementType": "i32" },
141
+ "a_main": { "scratch": "w_padded", "name": "a", "buffer": "read-only-storage", "elementType": "vec4<i32>" },
142
+ "a_scale": { "arg": "x_scale", "elementType": "f32", "length": 1 },
143
+ "b_scale": { "arg": "w_scale", "elementType": "f32" },
144
+ "y_scale": { "elementType": "f32", "length": 1 },
145
+ "y_zero_point": { "elementType": "u32", "length": 1 },
146
+ "y": { "elementType": "u32" },
147
  "params": {
 
148
  "struct": [
149
  { "name": "M", "type": "u32", "value": "convM" },
150
  { "name": "N", "type": "u32", "value": "convN" },
 
154
  { "name": "yBatchStride", "type": "u32", "value": "convYStride" }
155
  ]
156
  },
157
+ "x": { "elementType": "u32" },
158
+ "x_zero_point": { "elementType": "u32", "length": 1 },
159
+ "cols": { "scratch": "cols_padded", "elementType": "u32" },
160
+ "x_scale": { "elementType": "f32", "length": 1 },
161
+ "w": { "elementType": "i32" },
162
+ "w_scale": { "elementType": "f32" },
163
+ "w_zero_point": { "elementType": "i32" },
164
+ "a_w": { "arg": "w", "name": "a", "elementType": "$aVec4" },
165
+ "a_zero_point_main": { "arg": "w_zero_point", "name": "a_zero_point", "elementType": "$aScalar" },
166
+ "b_zero_point_main": { "arg": "x_zero_point", "name": "b_zero_point", "elementType": "$bScalar", "length": 1 },
167
+ "y_zero_point_main": { "name": "y_zero_point", "elementType": "$yScalar", "length": 1 },
168
+ "y_main": { "name": "y", "elementType": "$yScalar" },
169
+ "params_main": {
 
 
 
 
 
 
 
 
 
 
 
170
  "name": "params",
 
171
  "struct": [
172
  { "name": "M", "type": "u32", "value": "convM" },
173
  { "name": "N", "type": "u32", "value": "convN" },
 
177
  { "name": "yBatchStride", "type": "u32", "value": "convYStride" }
178
  ]
179
  },
180
+ "x_main": { "name": "x", "elementType": "$xScalar" },
181
+ "x_zero_point_main": { "name": "x_zero_point", "elementType": "$xScalar", "length": 1 },
182
+ "w_main": { "name": "w", "elementType": "$wScalar" },
183
+ "w_zero_point_main": { "name": "w_zero_point", "elementType": "$wScalar" },
184
+ "y_i32": { "scratch": "accum", "name": "y", "elementType": "i32" },
185
+ "params__uniform": {
186
  "name": "params",
 
187
  "struct": [
188
  { "name": "inChannels", "type": "u32", "value": "dim(shapes.x, 1)" },
189
  { "name": "inD", "type": "u32", "value": "dim(shapes.x, 2)" },
 
211
  { "name": "count", "type": "u32", "value": "numel(shapes.y)" }
212
  ]
213
  },
214
+ "params_accumulate": {
215
  "name": "params",
 
216
  "struct": [
217
  { "name": "inChannels", "type": "u32", "value": "dim(shapes.x, 1)" },
218
  { "name": "inH", "type": "u32", "value": 1 },
 
234
  { "name": "count", "type": "u32", "value": "numel(shapes.y)" }
235
  ]
236
  },
237
+ "params_nchw2d_scalar_quantization": {
238
  "name": "params",
 
239
  "struct": [
240
  { "name": "inChannels", "type": "u32", "value": "dim(shapes.x, 1)" },
241
  { "name": "inH", "type": "u32", "value": "dim(shapes.x, 2)" },
 
258
  ]
259
  },
260
  "accum": { "buffer": "read-only-storage", "elementType": "i32" },
261
+ "params_requantize": {
262
  "name": "params",
 
263
  "struct": [
264
  { "name": "count", "type": "u32", "value": "numel(shapes.y)" },
265
  { "name": "outChannels", "type": "u32", "value": "dim(shapes.w, 0)" },
266
  { "name": "outImageSize", "type": "u32", "value": "numel(shapes.y) / (dim(shapes.y, 0) * dim(shapes.y, 1))" }
267
  ]
268
  },
269
+ "params_count": { "name": "params", "struct": [{ "name": "count", "type": "u32", "value": "numel(shapes.y)" }] },
270
+ "B": { "elementType": "i32" },
271
+ "x_im2col": { "name": "x", "elementType": "$bScalar" },
272
+ "x_zero_point_im2col": { "name": "x_zero_point", "elementType": "$bScalar", "length": 1 },
273
+ "cols_im2col": { "scratch": "cols3d", "name": "cols", "elementType": "$bScalar" },
274
+ "params_im2col": {
 
 
 
 
275
  "name": "params",
 
276
  "struct": [
277
  { "name": "outCount", "type": "u32", "value": "dim(shapes.y, 2) * dim(shapes.y, 3) * dim(shapes.y, 4)" },
278
  { "name": "inChannels", "type": "u32", "value": "dim(shapes.x, 1)" },
 
288
  { "name": "outW", "type": "u32", "value": "dim(shapes.y, 4)" }
289
  ]
290
  },
291
+ "cols_b_scalar": { "name": "cols", "buffer": "storage", "elementType": "$bScalar" },
292
+ "params_conv_int_im2col_spatial": {
293
  "name": "params",
 
294
  "struct": [
295
  { "name": "outCount", "type": "u32", "value": "dim(shapes.y, 2) * dim(shapes.y, 3)" },
296
  { "name": "inChannels", "type": "u32", "value": "dim(shapes.x, 1)" },
 
300
  { "name": "outW", "type": "u32", "value": "dim(shapes.y, 3)" }
301
  ]
302
  },
303
+ "b_main": { "scratch": "cols3d", "name": "b", "buffer": "read-only-storage", "elementType": "$bScalar" },
304
+ "b_b_scalar": { "scratch": "cols", "name": "b", "buffer": "read-only-storage", "elementType": "$bScalar" },
305
+ "bias": { "arg": "B", "elementType": "i32" }
306
  },
307
  "variants": [
308
  {
309
  "id": "dp4a_pointwise_1x1_padded_k_baseline",
310
  "priority": 21,
311
  "when": ["packedFeature", "not present.B", "not device.features.has(\"shader-f16\")", "rank4TensorOk", "tensorDtypes.x == \"uint8\"", "tensorDtypes.w == \"int8\"", "tensorDtypes.y == \"uint8\"", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.x, 1) % 4 != 0", "dim(shapes.x, 1) >= 4", "dim(shapes.w, 0) >= 8", "dim(shapes.y, 2) * dim(shapes.y, 3) >= 256", "dim(shapes.w, 2) == 1", "dim(shapes.w, 3) == 1", "strideH == 1", "strideW == 1", "padTop == 0 and padBottom == 0", "padLeft == 0 and padRight == 0", "dim(shapes.w, 1) == dim(shapes.x, 1)", "dim(shapes.y, 2) == dim(shapes.x, 2)", "dim(shapes.y, 3) == dim(shapes.x, 3)", "paddedMatricesStorageOk", "dp4a2dDispatchOk"],
312
+ "derive": { "aPacked": true, "padB": true },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
313
  "intermediates": [
314
  { "id": "w_padded", "dtype": "int32", "shape": "[dim(shapes.w, 0) * ceilDiv(dim(shapes.x, 1), 4) * 4]" },
315
  {
 
352
  "name": "QLinearConv.Dp4aPointwisePaddedK",
353
  "shader": "quant-dp4a-matmul.wgsl.jinja",
354
  "bindings": [
355
+ "a_main",
356
  { "scratch": "x_padded", "name": "b", "buffer": "read-only-storage", "elementType": "u32" },
357
  "a_zero_point",
358
  "b_zero_point",
 
375
  "id": "dp4a_im2col_nchw_padded_k_baseline",
376
  "priority": 21,
377
  "when": ["packedFeature", "not present.B", "not device.features.has(\"shader-f16\")", "rank4TensorOk", "tensorDtypes.x == \"uint8\"", "tensorDtypes.w == \"int8\"", "tensorDtypes.y == \"uint8\"", "flexibleWeightQuantOk", "attrs.group == 1", "strideH >= 1", "strideW >= 1", "dilationH >= 1", "dilationW >= 1", "(dim(shapes.w, 2) > 1 or dim(shapes.w, 3) > 1)", "dim(shapes.w, 1) == dim(shapes.x, 1)", "dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) % 4 != 0", "dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) >= 64", "dim(shapes.w, 0) >= 8", "dim(shapes.y, 2) * dim(shapes.y, 3) >= 256", "numel(shapes.y) >= 8192", "output2dShapeOk", "paddedMatricesStorageOk", "paddedPreparationDispatchOk", "dp4a2dDispatchOk"],
378
+ "derive": { "aPacked": true, "padB": false },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
379
  "intermediates": [
380
  {
381
  "id": "w_padded",
 
468
  "name": "QLinearConv.Im2ColPaddedKDp4a",
469
  "shader": "quant-dp4a-matmul.wgsl.jinja",
470
  "bindings": [
471
+ "a_main",
472
  { "scratch": "cols_padded", "name": "b", "buffer": "read-only-storage", "elementType": "u32" },
473
  "a_zero_point",
474
  "b_zero_point",
 
525
  "priority": 20,
526
  "when": ["packedFeature", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "(device.features.has(\"shader-f16\") or dim(shapes.x, 1) % 4 == 0)", "dim(shapes.w, 2) == 1", "dim(shapes.w, 3) == 1", "strideH == 1", "strideW == 1", "padTop == 0 and padBottom == 0", "padLeft == 0 and padRight == 0", "dim(shapes.w, 1) == dim(shapes.x, 1)", "dim(shapes.y, 2) == dim(shapes.x, 2)", "dim(shapes.y, 3) == dim(shapes.x, 3)", "dp4a2dDispatchOk"],
527
  "derive": {
 
 
528
  "aVec4": "(\"vec4<u32>\" if tensorDtypes.w == \"uint8\" else \"vec4<i32>\") if dim(shapes.x, 1) % 4 == 0 else (\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\")",
529
+ "aPacked": "dim(shapes.x, 1) % 4 == 0"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
530
  },
531
  "passes": [
532
  {
 
534
  "name": "QLinearConv.Dp4aPointwiseFused",
535
  "shader": "quant-dp4a-matmul.wgsl.jinja",
536
  "bindings": [
537
+ "a_w",
538
  { "arg": "x", "name": "b", "elementType": "$bScalar" },
539
+ "a_zero_point_main",
540
+ "b_zero_point_main",
541
  "a_scale",
542
  "b_scale",
543
  "y_scale",
544
+ "y_zero_point_main",
545
+ "y_main",
546
+ "params_main"
547
  ],
548
  "dispatch": {
549
  "x": "ceil(dim(shapes.y, 2) * dim(shapes.y, 3) / 64)",
 
558
  "priority": 10,
559
  "when": ["not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "groupChannelsOk", "output2dShapeOk", "dim(shapes.y, 3) >= 4", "dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) >= 4", "numel(shapes.y) >= 32"],
560
  "derive": {
 
 
 
 
 
 
561
  "wScalePerChannel": "ranks.w_scale == 1 and dim(shapes.w_scale, 0) == dim(shapes.w, 0)",
562
  "wZeroPerChannel": "ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == dim(shapes.w, 0)",
563
+ "ocTile": "4 if dim(shapes.w, 0) >= attrs.group * 4 and dim(shapes.w, 0) % (attrs.group * 4) == 0 else 1"
 
 
564
  },
565
  "passes": [
566
  {
 
568
  "name": "QLinearConv.FeatureIndependentX4",
569
  "shader": "qlinear-conv-nchw-x4.wgsl.jinja",
570
  "bindings": [
571
+ "x_main",
572
  "x_scale",
573
+ "x_zero_point_main",
574
+ "w_main",
575
  "w_scale",
576
+ "w_zero_point_main",
577
  "y_scale",
578
+ "y_zero_point_main",
579
+ "y_main",
580
  {
581
  "name": "params",
582
  "struct": [
 
612
  {
613
  "id": "ncdhw3d_per_channel_quantization",
614
  "when": ["not present.B", "rank5TensorOk and dim(shapes.y, 2) == expectedOutputDepth and dim(shapes.y, 3) == expectedOutputHeight and dim(shapes.y, 4) == expectedOutputWidth", "not scalarWeightQuantOk and ranks.w_scale == 1 and dim(shapes.w_scale, 0) == dim(shapes.w, 0) and ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == dim(shapes.w, 0)", "groupChannelsOk"],
615
+ "derive": { "hasBias": false, "wScalePerChannel": true, "wZeroPerChannel": true },
 
 
 
 
 
 
 
 
 
 
 
 
616
  "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }],
617
  "passes": [
618
  {
 
620
  "name": "QLinearConv.AccumulatePerChannel3d",
621
  "shader": "conv-int-accumulate-spatial.wgsl.jinja",
622
  "derive": { "spatialRankSpec": 3 },
623
+ "bindings": ["x_main", "x_zero_point_main", "w_main", "w_zero_point_main", "y_i32", "params__uniform"],
624
  "dispatch": {
625
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
626
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
 
631
  "id": "requantize",
632
  "name": "QLinearConv.Requantize",
633
  "shader": "qlinear-conv-requantize.wgsl.jinja",
634
+ "bindings": ["accum", "x_scale", "w_scale", "y_scale", "y_zero_point_main", "y_main", "params_requantize"],
635
  "dispatch": {
636
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
637
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
 
643
  {
644
  "id": "ncw1d_scalar_quantization",
645
  "when": ["not present.B", "rank3TensorOk and dim(shapes.y, 2) == expectedOutputWidth", "scalarWeightQuantOk", "groupChannelsOk"],
646
+ "derive": { "hasBias": false },
 
 
 
 
 
 
 
 
 
 
647
  "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }],
648
  "passes": [
649
  {
 
651
  "name": "QLinearConv.Accumulate1d",
652
  "shader": "conv-int-accumulate-spatial.wgsl.jinja",
653
  "derive": { "spatialRankSpec": 2 },
654
+ "bindings": ["x_main", "x_zero_point_main", "w_main", "w_zero_point_main", "y_i32", "params_accumulate"],
655
  "dispatch": {
656
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
657
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
 
662
  "id": "requantize",
663
  "name": "QLinearConv.Requantize",
664
  "shader": "qlinear-conv-requantize.wgsl.jinja",
665
+ "bindings": ["accum", "x_scale", "w_scale", "y_scale", "y_zero_point_main", "y_main", "params_count"],
666
  "dispatch": {
667
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
668
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
 
674
  {
675
  "id": "ncdhw3d_scalar_quantization",
676
  "when": ["not present.B", "rank5TensorOk and dim(shapes.y, 2) == expectedOutputDepth and dim(shapes.y, 3) == expectedOutputHeight and dim(shapes.y, 4) == expectedOutputWidth", "scalarWeightQuantOk", "groupChannelsOk"],
677
+ "derive": { "hasBias": false },
 
 
 
 
 
 
 
 
 
 
678
  "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }],
679
  "passes": [
680
  {
 
682
  "name": "QLinearConv.Accumulate3d",
683
  "shader": "conv-int-accumulate-spatial.wgsl.jinja",
684
  "derive": { "spatialRankSpec": 3 },
685
+ "bindings": ["x_main", "x_zero_point_main", "w_main", "w_zero_point_main", "y_i32", "params__uniform"],
686
  "dispatch": {
687
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
688
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
 
693
  "id": "requantize",
694
  "name": "QLinearConv.Requantize",
695
  "shader": "qlinear-conv-requantize.wgsl.jinja",
696
+ "bindings": ["accum", "x_scale", "w_scale", "y_scale", "y_zero_point_main", "y_main", "params_count"],
697
  "dispatch": {
698
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
699
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
 
705
  {
706
  "id": "nchw2d_scalar_quantization",
707
  "when": ["not present.B", "rank4TensorOk and output2dShapeOk", "scalarWeightQuantOk", "groupChannelsOk"],
708
+ "derive": { "hasBias": false },
 
 
 
 
 
 
 
 
 
 
709
  "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }],
710
  "passes": [
711
  {
 
713
  "name": "QLinearConv.Accumulate",
714
  "shader": "conv-int-accumulate-spatial.wgsl.jinja",
715
  "derive": { "spatialRankSpec": 2 },
716
+ "bindings": ["x_main", "x_zero_point_main", "w_main", "w_zero_point_main", "y_i32", "params_nchw2d_scalar_quantization"],
717
  "dispatch": {
718
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
719
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
 
724
  "id": "requantize",
725
  "name": "QLinearConv.Requantize",
726
  "shader": "qlinear-conv-requantize.wgsl.jinja",
727
+ "bindings": ["accum", "x_scale", "w_scale", "y_scale", "y_zero_point_main", "y_main", "params_count"],
728
  "dispatch": {
729
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
730
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
 
736
  {
737
  "id": "ncw1d_scalar_quantization_bias",
738
  "when": ["present.B and ranks.B == 1 and dim(shapes.B, 0) == dim(shapes.w, 0)", "rank3TensorOk and dim(shapes.y, 2) == expectedOutputWidth", "scalarWeightQuantOk", "groupChannelsOk"],
739
+ "derive": { "hasBias": true },
 
 
 
 
 
 
 
 
 
 
740
  "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }],
741
  "passes": [
742
  {
 
744
  "name": "QLinearConv.Accumulate1d",
745
  "shader": "conv-int-accumulate-spatial.wgsl.jinja",
746
  "derive": { "spatialRankSpec": 2 },
747
+ "bindings": ["x_main", "x_zero_point_main", "w_main", "w_zero_point_main", "y_i32", "params_accumulate"],
748
  "dispatch": {
749
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
750
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
 
755
  "id": "requantize",
756
  "name": "QLinearConv.Requantize",
757
  "shader": "qlinear-conv-requantize.wgsl.jinja",
758
+ "bindings": ["accum", "x_scale", "w_scale", "y_scale", "y_zero_point_main", "B", "y_main", "params_requantize"],
759
  "dispatch": {
760
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
761
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
 
767
  {
768
  "id": "ncdhw3d_scalar_quantization_bias",
769
  "when": ["present.B and ranks.B == 1 and dim(shapes.B, 0) == dim(shapes.w, 0)", "rank5TensorOk and dim(shapes.y, 2) == expectedOutputDepth and dim(shapes.y, 3) == expectedOutputHeight and dim(shapes.y, 4) == expectedOutputWidth", "scalarWeightQuantOk", "groupChannelsOk"],
770
+ "derive": { "hasBias": true },
 
 
 
 
 
 
 
 
 
 
771
  "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }],
772
  "passes": [
773
  {
 
775
  "name": "QLinearConv.Accumulate3d",
776
  "shader": "conv-int-accumulate-spatial.wgsl.jinja",
777
  "derive": { "spatialRankSpec": 3 },
778
+ "bindings": ["x_main", "x_zero_point_main", "w_main", "w_zero_point_main", "y_i32", "params__uniform"],
779
  "dispatch": {
780
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
781
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
 
786
  "id": "requantize",
787
  "name": "QLinearConv.Requantize",
788
  "shader": "qlinear-conv-requantize.wgsl.jinja",
789
+ "bindings": ["accum", "x_scale", "w_scale", "y_scale", "y_zero_point_main", "B", "y_main", "params_requantize"],
790
  "dispatch": {
791
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
792
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
 
798
  {
799
  "id": "ncw1d_per_channel_quantization",
800
  "when": ["not present.B", "rank3TensorOk and dim(shapes.y, 2) == expectedOutputWidth", "not scalarWeightQuantOk and ranks.w_scale == 1 and dim(shapes.w_scale, 0) == dim(shapes.w, 0) and ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == dim(shapes.w, 0)", "groupChannelsOk"],
801
+ "derive": { "hasBias": false, "wScalePerChannel": true, "wZeroPerChannel": true },
 
 
 
 
 
 
 
 
 
 
 
 
802
  "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }],
803
  "passes": [
804
  {
 
806
  "name": "QLinearConv.AccumulatePerChannel1d",
807
  "shader": "conv-int-accumulate-spatial.wgsl.jinja",
808
  "derive": { "spatialRankSpec": 2 },
809
+ "bindings": ["x_main", "x_zero_point_main", "w_main", "w_zero_point_main", "y_i32", "params_accumulate"],
810
  "dispatch": {
811
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
812
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
 
817
  "id": "requantize",
818
  "name": "QLinearConv.Requantize",
819
  "shader": "qlinear-conv-requantize.wgsl.jinja",
820
+ "bindings": ["accum", "x_scale", "w_scale", "y_scale", "y_zero_point_main", "y_main", "params_requantize"],
821
  "dispatch": {
822
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
823
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
 
831
  "priority": 20,
832
  "when": ["packedFeature", "not present.B", "rank5TensorOk", "attrs.group == 1", "dim(shapes.w, 1) == dim(shapes.x, 1)", "dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) * dim(shapes.w, 4) % 4 == 0", "dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) * dim(shapes.w, 4) >= 32", "dim(shapes.w, 0) >= 8", "dim(shapes.y, 2) * dim(shapes.y, 3) * dim(shapes.y, 4) >= 256", "flexibleWeightQuantOk", "dim(shapes.y, 2) == expectedOutputDepth", "dim(shapes.y, 3) == expectedOutputHeight", "dim(shapes.y, 4) == expectedOutputWidth", "im2colStorageOk", "dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) * dim(shapes.w, 4) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "ceilDiv(dim(shapes.y, 2) * dim(shapes.y, 3) * dim(shapes.y, 4), tunables.WORKGROUP_SIZE) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "ceilDiv(dim(shapes.y, 2) * dim(shapes.y, 3) * dim(shapes.y, 4), 64) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "ceilDiv(dim(shapes.w, 0), 8) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "dim(shapes.x, 0) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)"],
833
  "derive": {
 
 
834
  "aVec4": "\"vec4<u32>\" if tensorDtypes.w == \"uint8\" else \"vec4<i32>\"",
 
 
 
 
 
 
835
  "aPacked": true,
 
 
 
 
 
 
 
 
 
836
  "hasBias": false
837
  },
838
  "intermediates": [
 
862
  "padTopSpec": "effectivePadTop",
863
  "padLeftSpec": "effectivePadLeft"
864
  },
865
+ "bindings": ["x_im2col", "x_zero_point_im2col", "cols_im2col", "params_im2col"],
866
  "dispatch": {
867
  "x": "ceil(dim(shapes.y, 2) * dim(shapes.y, 3) * dim(shapes.y, 4) / tunables.WORKGROUP_SIZE)",
868
  "y": "dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) * dim(shapes.w, 4)",
 
873
  "id": "main",
874
  "name": "QLinearConv.Im2Col3dDp4a",
875
  "shader": "quant-dp4a-matmul.wgsl.jinja",
876
+ "bindings": ["a_w", "b_main", "a_zero_point_main", "b_zero_point_main", "a_scale", "b_scale", "y_scale", "y_zero_point_main", "y_main", "params_main"],
877
  "dispatch": {
878
  "x": "ceil(dim(shapes.y, 2) * dim(shapes.y, 3) * dim(shapes.y, 4) / 64)",
879
  "y": "ceil(dim(shapes.w, 0) / 8)",
 
887
  "priority": 25,
888
  "when": ["packedFeature", "present.B", "ranks.B == 1", "dim(shapes.B, 0) == dim(shapes.w, 0)", "rank4TensorOk", "attrs.group == 1", "dim(shapes.w, 1) == dim(shapes.x, 1)", "flexibleWeightQuantOk"],
889
  "derive": {
 
 
890
  "aVec4": "(\"vec4<u32>\" if tensorDtypes.w == \"uint8\" else \"vec4<i32>\") if dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) % 4 == 0 else (\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\")",
 
 
 
 
 
 
891
  "aPacked": "dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) % 4 == 0",
 
 
 
 
 
 
 
 
 
892
  "hasBias": true
893
  },
894
  "intermediates": [
 
914
  "padTopSpec": "effectivePadTop",
915
  "padLeftSpec": "effectivePadLeft"
916
  },
917
+ "bindings": ["x_im2col", "x_zero_point_im2col", "cols_b_scalar", "params_conv_int_im2col_spatial"],
918
  "dispatch": {
919
  "x": "ceil(dim(shapes.y, 2) * dim(shapes.y, 3) / tunables.WORKGROUP_SIZE)",
920
  "y": "dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3)",
 
925
  "id": "main",
926
  "name": "QLinearConv.Im2ColDp4aBias",
927
  "shader": "quant-dp4a-matmul.wgsl.jinja",
928
+ "bindings": ["a_w", "b_b_scalar", "a_zero_point_main", "b_zero_point_main", "a_scale", "b_scale", "y_scale", "y_zero_point_main", "bias", "y_main", "params_main"],
929
  "dispatch": {
930
  "x": "ceil(dim(shapes.y, 2) * dim(shapes.y, 3) / 64)",
931
  "y": "ceil(dim(shapes.w, 0) / 8)",
 
939
  "priority": 20,
940
  "when": ["packedFeature", "not present.B", "flexibleWeightQuantOk", "rank4TensorOk", "attrs.group == 1", "(device.features.has(\"shader-f16\") or dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) % 4 == 0)", "(dim(shapes.w, 2) > 1 or dim(shapes.w, 3) > 1)", "dim(shapes.w, 1) == dim(shapes.x, 1)", "dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) >= 64", "numel(shapes.y) >= 8192", "dp4a2dDispatchOk"],
941
  "derive": {
 
 
942
  "aVec4": "(\"vec4<u32>\" if tensorDtypes.w == \"uint8\" else \"vec4<i32>\") if dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) % 4 == 0 else (\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\")",
943
+ "aPacked": "dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3) % 4 == 0"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
944
  },
945
  "intermediates": [
946
  {
 
965
  "padTopSpec": "effectivePadTop",
966
  "padLeftSpec": "effectivePadLeft"
967
  },
968
+ "bindings": ["x_im2col", "x_zero_point_im2col", "cols_b_scalar", "params_conv_int_im2col_spatial"],
969
  "dispatch": {
970
  "x": "ceil(dim(shapes.y, 2) * dim(shapes.y, 3) / tunables.WORKGROUP_SIZE)",
971
  "y": "dim(shapes.w, 1) * dim(shapes.w, 2) * dim(shapes.w, 3)",
 
976
  "id": "main",
977
  "name": "QLinearConv.Im2ColDp4a",
978
  "shader": "quant-dp4a-matmul.wgsl.jinja",
979
+ "bindings": ["a_w", "b_b_scalar", "a_zero_point_main", "b_zero_point_main", "a_scale", "b_scale", "y_scale", "y_zero_point_main", "y_main", "params_main"],
980
  "dispatch": {
981
  "x": "ceil(dim(shapes.y, 2) * dim(shapes.y, 3) / 64)",
982
  "y": "ceil(dim(shapes.w, 0) / 8)",
 
989
  "id": "nchw2d_per_channel_quantization",
990
  "priority": 5,
991
  "when": ["not present.B", "rank4TensorOk", "ranks.w_scale == 1", "dim(shapes.w_scale, 0) == dim(shapes.w, 0)", "ranks.w_zero_point == 1", "dim(shapes.w_zero_point, 0) == dim(shapes.w, 0)", "groupChannelsOk", "output2dShapeOk"],
992
+ "derive": { "hasBias": false, "wScalePerChannel": true, "wZeroPerChannel": true },
 
 
 
 
 
 
 
 
 
 
 
 
993
  "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }],
994
  "passes": [
995
  {
 
997
  "name": "QLinearConv.AccumulatePerChannel",
998
  "shader": "conv-int-accumulate-spatial.wgsl.jinja",
999
  "derive": { "spatialRankSpec": 2 },
1000
+ "bindings": ["x_main", "x_zero_point_main", "w_main", "w_zero_point_main", "y_i32", "params_nchw2d_scalar_quantization"],
1001
  "dispatch": {
1002
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
1003
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
 
1008
  "id": "requantize",
1009
  "name": "QLinearConv.RequantizePerChannel",
1010
  "shader": "qlinear-conv-requantize.wgsl.jinja",
1011
+ "bindings": ["accum", "x_scale", "w_scale", "y_scale", "y_zero_point_main", "y_main", "params_requantize"],
1012
  "dispatch": {
1013
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
1014
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
 
1020
  {
1021
  "id": "nchw2d_per_channel_quantization_bias",
1022
  "when": ["present.B", "ranks.B == 1", "dim(shapes.B, 0) == dim(shapes.w, 0)", "rank4TensorOk", "ranks.w_scale == 1", "dim(shapes.w_scale, 0) == dim(shapes.w, 0)", "ranks.w_zero_point == 1", "dim(shapes.w_zero_point, 0) == dim(shapes.w, 0)", "groupChannelsOk", "output2dShapeOk"],
1023
+ "derive": { "hasBias": true, "wScalePerChannel": true, "wZeroPerChannel": true },
 
 
 
 
 
 
 
 
 
 
 
 
1024
  "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }],
1025
  "passes": [
1026
  {
 
1028
  "name": "QLinearConv.AccumulatePerChannel",
1029
  "shader": "conv-int-accumulate-spatial.wgsl.jinja",
1030
  "derive": { "spatialRankSpec": 2 },
1031
+ "bindings": ["x_main", "x_zero_point_main", "w_main", "w_zero_point_main", "y_i32", "params_nchw2d_scalar_quantization"],
1032
  "dispatch": {
1033
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
1034
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
 
1039
  "id": "requantize",
1040
  "name": "QLinearConv.RequantizePerChannelBias",
1041
  "shader": "qlinear-conv-requantize.wgsl.jinja",
1042
+ "bindings": ["accum", "x_scale", "w_scale", "y_scale", "y_zero_point_main", "B", "y_main", "params_requantize"],
1043
  "dispatch": {
1044
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
1045
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
 
1051
  {
1052
  "id": "nchw2d_scalar_quantization_bias",
1053
  "when": ["present.B", "ranks.B == 1", "dim(shapes.B, 0) == dim(shapes.w, 0)", "rank4TensorOk", "scalarWeightQuantOk", "groupChannelsOk", "output2dShapeOk"],
1054
+ "derive": { "hasBias": true },
 
 
 
 
 
 
 
 
 
 
1055
  "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }],
1056
  "passes": [
1057
  {
 
1059
  "name": "QLinearConv.Accumulate",
1060
  "shader": "conv-int-accumulate-spatial.wgsl.jinja",
1061
  "derive": { "spatialRankSpec": 2 },
1062
+ "bindings": ["x_main", "x_zero_point_main", "w_main", "w_zero_point_main", "y_i32", "params_nchw2d_scalar_quantization"],
1063
  "dispatch": {
1064
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
1065
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
 
1070
  "id": "requantize",
1071
  "name": "QLinearConv.RequantizeBias",
1072
  "shader": "qlinear-conv-requantize.wgsl.jinja",
1073
+ "bindings": ["accum", "x_scale", "w_scale", "y_scale", "y_zero_point_main", "B", "y_main", "params_requantize"],
1074
  "dispatch": {
1075
  "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
1076
  "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)",
build/webgpu/metadata.json CHANGED
@@ -1,27 +1,27 @@
1
  {
2
  "name": "ai.onnx.QLinearConv",
3
- "id": "_ai_onnx_qlinearconv_webgpu_5dc8940",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "backend": { "type": "webgpu" },
7
  "digest": {
8
  "algorithm": "sha256",
9
  "files": {
10
- "bench.json": "spUQwS0Tr+3Vdfkxd9WepuT6g6ezyeV6aMao4r7OsdY=",
11
- "conv-int-accumulate-spatial.wgsl.jinja": "TkacF0/fJ1yShdyPJWePK/+8XJFaV+ZDKWT6XvHOCI8=",
12
- "conv-int-im2col-spatial.wgsl.jinja": "3vFeBqBVtyYtxV1yteSQ3O7skFur08E3vDoVtdwcYAo=",
13
- "manifest.json": "VInAZzgFiEXRlHexx+MCXfvzy6hnwIGSsbdY6pw6Nas=",
14
- "qlinear-conv-dp4a-direct-3x3.wgsl.jinja": "xt3DqIwhlzomIiLMuMZ+8F0+Xqg7AWfGCbeZnNT2rLU=",
15
- "qlinear-conv-nchw-x4.wgsl.jinja": "XT1MIv1iIEOhj0keQUkU5cau6Nc/CTJCAgu+PardWaQ=",
16
- "qlinear-conv-requantize.wgsl.jinja": "JCGp/e1wWm22EQlsYPNreGtdualHITd5dmvA9kGYZCk=",
17
- "quant-dp4a-matmul.wgsl.jinja": "x1yY2y4Wb6vlzMVeRw+DbEmfvJdPBHkMF8ATG34ZGis=",
18
- "quant-pad-k-matrices.wgsl.jinja": "hsLWgSTbpXBnsIOyrIzsmhQzVyIYALaFvx62VN46KyY=",
19
- "test.json": "z2krhaHYtSozWBIW2J1ZKnlpd1S7dDdTQwYq83sf/7g="
20
  }
21
  },
22
- "provenance": { "kernel": { "sha": "91d990483a174128daf7673f3f37a7c890493ae1", "dirty": false } },
23
  "webgpu": {
24
- "manifestSpec": "2.0",
25
  "variants": {
26
  "dp4a_pointwise_1x1_padded_k_baseline": ["quant-dp4a-matmul.wgsl.jinja", "quant-pad-k-matrices.wgsl.jinja"],
27
  "dp4a_im2col_nchw_padded_k_baseline": ["conv-int-im2col-spatial.wgsl.jinja", "quant-dp4a-matmul.wgsl.jinja", "quant-pad-k-matrices.wgsl.jinja"],
 
1
  {
2
  "name": "ai.onnx.QLinearConv",
3
+ "id": "_ai_onnx_qlinearconv_webgpu_a9dd212",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "backend": { "type": "webgpu" },
7
  "digest": {
8
  "algorithm": "sha256",
9
  "files": {
10
+ "bench.json": "dImz59rzdOP1XQWQNeiGwVd5UfcX//dFlJq9a+5lnNw=",
11
+ "conv-int-accumulate-spatial.wgsl.jinja": "7393dtsdsogF8Zzus4+gZiwygpXHb15ZTWhE9VM83UA=",
12
+ "conv-int-im2col-spatial.wgsl.jinja": "P+nJ/PAP2Qv0SfY7wyVmNCDDHXRRcR/Uv4Pe2sowiqo=",
13
+ "manifest.json": "xHXn1ZxuY2tq/rvmMjxkxVyLryGDDke5HrvjnAS/Ya4=",
14
+ "qlinear-conv-dp4a-direct-3x3.wgsl.jinja": "SNVVWL7lOnDFHm6eHR1eXAYy5zRIc4vqKYWEl4cAdt4=",
15
+ "qlinear-conv-nchw-x4.wgsl.jinja": "CrFYcoL6MLszFVzk877uUZu6eMi7h6Wii63GJXWC+x0=",
16
+ "qlinear-conv-requantize.wgsl.jinja": "ElB8XipqRr6CjNjGakSOhsdsM/TezfpRQ2zyh+vENPQ=",
17
+ "quant-dp4a-matmul.wgsl.jinja": "1C5bi3ZyBymNsP6SymoruNP6hImWtSGJgcIjE0WKgb4=",
18
+ "quant-pad-k-matrices.wgsl.jinja": "Kuq0pV3hrj1M8etjKsJTBM8WmP12d8WNZVwhYbSIJyQ=",
19
+ "test.json": "fCaJYVZme0lNciPQNNq45ORkoxPwg+sS7axeDvKCsDo="
20
  }
21
  },
22
+ "provenance": { "kernel": { "sha": "6fdf6301e2bbcc2f03bf1eaf493b7ad55ef33afc", "dirty": false } },
23
  "webgpu": {
24
+ "manifestSpec": "2.1",
25
  "variants": {
26
  "dp4a_pointwise_1x1_padded_k_baseline": ["quant-dp4a-matmul.wgsl.jinja", "quant-pad-k-matrices.wgsl.jinja"],
27
  "dp4a_im2col_nchw_padded_k_baseline": ["conv-int-im2col-spatial.wgsl.jinja", "quant-dp4a-matmul.wgsl.jinja", "quant-pad-k-matrices.wgsl.jinja"],
build/webgpu/qlinear-conv-dp4a-direct-3x3.wgsl.jinja CHANGED
@@ -1,3 +1,11 @@
 
 
 
 
 
 
 
 
1
  // Direct tiled u8 x s8 3x3 convolution using packed 4-lane integer dots.
2
  //
3
  // An 8x8 output workgroup cooperatively stages a 10x10 halo for sixteen input
@@ -11,7 +19,261 @@
11
  fn dot4_packed(a_word: u32, b_word: u32) -> i32 {
12
  return dot4I8Packed(a_word, b_word);
13
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
15
 
16
  const IN_C: u32 = {{ inChannels }}u;
17
  const IN_H: u32 = {{ inHeight }}u;
@@ -61,13 +323,7 @@ fn sum4_packed(v: u32) -> i32 {
61
  + signed_byte(v, 16u) + signed_byte(v, 24u);
62
  }
63
 
64
- fn read_w_zero({% if wZeroPerChannel %}oc: u32{% endif %}) -> i32 {
65
- {% if wZeroPerChannel %}
66
- return w_zero_point[oc];
67
- {% else %}
68
- return w_zero_point[0];
69
- {% endif %}
70
- }
71
 
72
  fn read_w_scale({% if wScalePerChannel %}oc: u32{% endif %}) -> f32 {
73
  {% if wScalePerChannel %}
@@ -78,9 +334,8 @@ fn read_w_scale({% if wScalePerChannel %}oc: u32{% endif %}) -> f32 {
78
  }
79
 
80
  fn requantize(acc: i32{% if wScalePerChannel %}, oc: u32{% endif %}) -> u32 {
81
- let scaled = f32(acc) * x_scale[0] * read_w_scale({% if wScalePerChannel %}oc{% endif %}) / y_scale[0];
82
  let q = clamp(
83
- i32(clamp(round(scaled), -2.0e9, 2.0e9)) + i32(y_zero_point[0]),
84
  0,
85
  255,
86
  );
 
1
+ {% macro quant_read_column_zero(fn_name, buffer_name, is_unsigned, present=true, per_column=false, param_name="col", return_type="i32") %}
2
+ fn {{ fn_name }}({% if per_column %}{{ param_name }}: u32{% endif %}) -> {{ return_type }} {
3
+ {% if per_column %}
4
+ return {{ buffer_name }}[{{ param_name }}];
5
+ {% else %}
6
+ return {{ buffer_name }}[0];
7
+ {% endif %}
8
+ }{% endmacro %}
9
  // Direct tiled u8 x s8 3x3 convolution using packed 4-lane integer dots.
10
  //
11
  // An 8x8 output workgroup cooperatively stages a 10x10 halo for sixteen input
 
19
  fn dot4_packed(a_word: u32, b_word: u32) -> i32 {
20
  return dot4I8Packed(a_word, b_word);
21
  }
22
+ // Rare-boundary fallback: reproduce the reference's binary64 scale division
23
+ // and accumulator product using integer significands; no f64 feature required.
24
+ fn quant_u96_add(a: vec3<u32>, b: vec3<u32>) -> vec3<u32> {
25
+ let low = a.x + b.x;
26
+ let middle = a.y + b.y;
27
+ let with_carry = middle + u32(low < a.x);
28
+ return vec3<u32>(low, with_carry, a.z + b.z + u32(middle < a.y) + u32(with_carry < middle));
29
+ }
30
+ fn quant_u96_shr(a: vec3<u32>) -> vec3<u32> {
31
+ return vec3<u32>((a.x >> 1u) | (a.y << 31u), (a.y >> 1u) | (a.z << 31u), a.z >> 1u);
32
+ }
33
+ fn quant_u96_round_shift(value: vec3<u32>, shift: u32) -> vec3<u32> {
34
+ var q = value;
35
+ var sticky = false;
36
+ var guard = false;
37
+ for (var bit = 0u; bit < shift; bit++) {
38
+ sticky = sticky || guard;
39
+ guard = (q.x & 1u) != 0u;
40
+ q = quant_u96_shr(q);
41
+ }
42
+ if (guard && (sticky || (q.x & 1u) != 0u)) {
43
+ q = quant_u96_add(q, vec3<u32>(1u, 0u, 0u));
44
+ }
45
+ return q;
46
+ }
47
+ fn quant_float_significand(bits: u32) -> u32 {
48
+ return (bits & 0x7fffffu) | select(0x800000u, 0u, (bits & 0x7f800000u) == 0u);
49
+ }
50
+ fn quant_float_exponent(bits: u32) -> i32 {
51
+ return max(1, i32((bits >> 23u) & 255u)) - 150;
52
+ }
53
+ fn quant_round_binary64(acc: i32, a_s: f32, b_s: f32, y_s: f32) -> i32 {
54
+ if (acc == 0) { return 0; }
55
+ let a_bits = bitcast<u32>(a_s);
56
+ let b_bits = bitcast<u32>(b_s);
57
+ let y_bits = bitcast<u32>(y_s);
58
+ let a = quant_float_significand(a_bits);
59
+ let b = quant_float_significand(b_bits);
60
+ let denominator = quant_float_significand(y_bits);
61
+ if (a == 0u || b == 0u) { return 0; }
62
+ // The exact product of two 24-bit significands occupies at most 48 bits.
63
+ let p0 = (a & 65535u) * (b & 65535u);
64
+ let cross = (a >> 16u) * (b & 65535u) + (b >> 16u) * (a & 65535u);
65
+ let product_low = p0 + (cross << 16u);
66
+ let product_high = (a >> 16u) * (b >> 16u) + (cross >> 16u) + u32(product_low < p0);
67
+ var input_bit = select(31 - i32(countLeadingZeros(product_low)), 63 - i32(countLeadingZeros(product_high)), product_high != 0u);
68
+ var remainder = 0u;
69
+ var significant_bits = 0u;
70
+ var leading_exponent = 0;
71
+ var significand = vec2<u32>(0u);
72
+ // Long division generates the 53 bits of the binary64 multiplier. The
73
+ // remainder fits in 25 bits, including its left shift. At most 24 leading
74
+ // zero bits precede the first significant bit.
75
+ loop {
76
+ var incoming = 0u;
77
+ if (input_bit >= 32) { incoming = (product_high >> u32(input_bit - 32)) & 1u; }
78
+ else if (input_bit >= 0) { incoming = (product_low >> u32(input_bit)) & 1u; }
79
+ remainder = (remainder << 1u) | incoming;
80
+ let digit = u32(remainder >= denominator);
81
+ if (digit != 0u) { remainder -= denominator; }
82
+ if (significant_bits != 0u || digit != 0u) {
83
+ if (significant_bits == 0u) { leading_exponent = input_bit; }
84
+ significand = vec2<u32>((significand.x << 1u) | digit, (significand.y << 1u) | (significand.x >> 31u));
85
+ significant_bits++;
86
+ }
87
+ input_bit--;
88
+ if (significant_bits == 53u) { break; }
89
+ }
90
+ let twice_remainder = remainder << 1u;
91
+ if (twice_remainder > denominator || (twice_remainder == denominator && (significand.x & 1u) != 0u)) {
92
+ let low = significand.x + 1u;
93
+ significand = vec2<u32>(low, significand.y + u32(low == 0u));
94
+ }
95
+ let scale_exponent = quant_float_exponent(a_bits) + quant_float_exponent(b_bits) - quant_float_exponent(y_bits) + leading_exponent - 52;
96
+ // abs(INT_MIN) is representable as u32. Multiply the rounded multiplier's
97
+ // significand by the exact accumulator into three little-endian u32 limbs.
98
+ var magnitude = select(u32(acc), 0u - u32(acc), acc < 0);
99
+ var operand = vec3<u32>(significand, 0u);
100
+ var product = vec3<u32>(0u);
101
+ loop {
102
+ if ((magnitude & 1u) != 0u) { product = quant_u96_add(product, operand); }
103
+ magnitude >>= 1u;
104
+ if (magnitude == 0u) { break; }
105
+ operand = vec3<u32>(operand.x << 1u, (operand.y << 1u) | (operand.x >> 31u), (operand.z << 1u) | (operand.y >> 31u));
106
+ }
107
+ var product_top = 31 - i32(countLeadingZeros(product.x));
108
+ if (product.y != 0u) { product_top = 63 - i32(countLeadingZeros(product.y)); }
109
+ if (product.z != 0u) { product_top = 95 - i32(countLeadingZeros(product.z)); }
110
+ if (product_top + scale_exponent >= 10) { return select(1024, -1024, acc < 0); }
111
+ if (product_top + scale_exponent < -2) { return 0; }
112
+ let product_shift = u32(max(0, product_top - 52));
113
+ let rounded_product = quant_u96_round_shift(product, product_shift);
114
+ let result_exponent = scale_exponent + i32(product_shift);
115
+ var rounded = rounded_product;
116
+ if (result_exponent < 0) { rounded = quant_u96_round_shift(rounded_product, u32(-result_exponent)); }
117
+ else { rounded.x <<= u32(result_exponent); }
118
+ return select(i32(rounded.x), -i32(rounded.x), acc < 0);
119
+ }
120
+
121
+ // A power-of-two division and an exactly represented <=48-bit product need
122
+ // no binary64 boundary resolver. This includes ordinary dyadic quantization.
123
+ fn quant_exact_dyadic_product(acc: i32, scale: f32) -> bool {
124
+ let magnitude = select(u32(acc), 0u - u32(acc), acc < 0);
125
+ let significand = quant_float_significand(bitcast<u32>(scale));
126
+ let integer_bits = 32u - countLeadingZeros(magnitude) - countTrailingZeros(magnitude);
127
+ let scale_bits = 32u - countLeadingZeros(significand) - countTrailingZeros(significand);
128
+ return integer_bits + scale_bits <= 48u;
129
+ }
130
+ fn quant_normal_expansion_range(v: f32) -> bool {
131
+ let exponent = (bitcast<u32>(v) >> 23u) & 255u;
132
+ // Reserve two binary32 significands plus four carry/error bits above the
133
+ // minimum normal exponent, so residual arithmetic avoids FTZ underflow.
134
+ return exponent >= (1u + 2u * 24u + 4u) && exponent < 255u;
135
+ }
136
+
137
+ fn round_expansion_ratio(numerator_hi: f32, numerator_lo: f32, denominator: f32, dyadic_scale: f32, dyadic_candidate: bool, acc: i32, a_s: f32, b_s: f32, y_s: f32) -> i32 {
138
+ let estimate = numerator_hi / denominator;
139
+ // Anything outside this range saturates after adding an 8-bit zero point;
140
+ // returning early also keeps floor/i32 conversion away from Inf and overflow.
141
+ if (estimate <= -1024.0) { return -1024; }
142
+ if (estimate >= 1024.0) { return 1024; }
143
+
144
+ let lo = floor(estimate);
145
+ let hi = lo + 1.0;
146
+ let midpoint = lo + 0.5;
147
+ let midpoint_hi = fma(midpoint, denominator, 0.0);
148
+ if ((bitcast<u32>(midpoint_hi) & 0x7f800000u) == 0x7f800000u) { return quant_round_binary64(acc, a_s, b_s, y_s); }
149
+ let midpoint_lo = fma(midpoint, denominator, -midpoint_hi);
150
+ // The high-word subtraction is exact for these similarly sized products
151
+ // (Sterbenz); adding the residual difference therefore preserves its sign
152
+ // even when the rounded high words straddle adjacent f32 values.
153
+ let midpoint_delta = fma(1.0, numerator_hi - midpoint_hi, numerator_lo - midpoint_lo);
154
+ // Escalate ambiguous expansion signs instead of changing the tie tolerance.
155
+ // Six FMA/product tails and their additions fit within this conservative
156
+ // 16*u^2 error envelope (u=2^-24), including binary64 reference rounding.
157
+ if ((bitcast<u32>(midpoint_delta) & 0x7f800000u) == 0x7f800000u) { return quant_round_binary64(acc, a_s, b_s, y_s); }
158
+ if (abs(midpoint_delta) <= max(abs(numerator_hi), abs(midpoint_hi)) * 0x1p-44) {
159
+ if (!dyadic_candidate || !quant_exact_dyadic_product(acc, dyadic_scale)) { return quant_round_binary64(acc, a_s, b_s, y_s); }
160
+ }
161
+ if (midpoint_delta < 0.0) {
162
+ return i32(lo);
163
+ }
164
+ if (midpoint_delta > 0.0) {
165
+ return i32(hi);
166
+ }
167
+ let half_lo = floor(lo * 0.5);
168
+ let lo_is_even = lo - half_lo * 2.0 == 0.0;
169
+ return i32(select(hi, lo, lo_is_even));
170
+ }
171
+
172
+ // Multiplication by a power of two keeps every i32 bit in binary64. Round the
173
+ // exact dyadic value directly, including INT_MIN, half-even ties and saturation.
174
+ fn quant_round_power_two(acc: i32, exponent: i32) -> i32 {
175
+ let magnitude = select(u32(acc), 0u - u32(acc), acc < 0);
176
+ var rounded = 0u;
177
+ if (exponent >= 0) {
178
+ if (exponent >= 10) { return select(1024, -1024, acc < 0); }
179
+ rounded = min(magnitude, 1024u >> u32(exponent)) << u32(exponent);
180
+ } else {
181
+ let shift = u32(-exponent);
182
+ if (shift >= 32u) { return 0; }
183
+ let whole = magnitude >> shift;
184
+ let remainder = magnitude & ((1u << shift) - 1u);
185
+ let half = 1u << (shift - 1u);
186
+ rounded = min(1024u, whole + u32(remainder > half || (remainder == half && (whole & 1u) != 0u)));
187
+ }
188
+ return select(i32(rounded), -i32(rounded), acc < 0);
189
+ }
190
+
191
+ fn round_requantized_precise(acc: i32, a_s: f32, b_s: f32, y_s: f32) -> i32 {
192
+ if (acc == 0) { return 0; }
193
+ let a_bits = bitcast<u32>(a_s);
194
+ let b_bits = bitcast<u32>(b_s);
195
+ let y_bits = bitcast<u32>(y_s);
196
+ if (((a_bits | b_bits | y_bits) & 0x807fffffu) == 0u && min(a_bits, min(b_bits, y_bits)) != 0u && max(a_bits, max(b_bits, y_bits)) < 0x7f800000u) {
197
+ let exponent = i32(a_bits >> 23u) + i32(b_bits >> 23u) - i32(y_bits >> 23u) - 127;
198
+ return quant_round_power_two(acc, exponent);
199
+ }
200
+ let acc_f = f32(acc);
201
+ // Every i32 high part aligned to 256 is exactly representable in f32.
202
+ // Recover the conversion residual without an out-of-range i32(acc_f)
203
+ // conversion when INT_MAX rounds up to 2^31.
204
+ let acc_tail = fma(-1.0, acc_f - f32(acc & -256), f32(acc & 255));
205
+ // Equal b/y scales cancel exactly. Bypass their product and division so a
206
+ // rounded intermediate cannot move a near-half value onto the tie.
207
+ if (bitcast<u32>(b_s) == bitcast<u32>(y_s)) {
208
+ let direct_hi = fma(acc_f, a_s, 0.0);
209
+ let direct_lo = fma(acc_tail, a_s, fma(acc_f, a_s, -direct_hi));
210
+ if (!quant_normal_expansion_range(direct_hi)) { return quant_round_binary64(acc, a_s, b_s, y_s); }
211
+ return round_expansion_ratio(direct_hi, direct_lo, 1.0, a_s, true, acc, a_s, b_s, y_s);
212
+ }
213
 
214
+ // Keep the scale product as a two-float expansion. Its FMA residual preserves
215
+ // which side of a rounding midpoint a near-half value occupies.
216
+ let ab_hi = fma(a_s, b_s, 0.0);
217
+ let ab_lo = fma(a_s, b_s, -ab_hi);
218
+ if (!quant_normal_expansion_range(ab_hi) || !quant_normal_expansion_range(y_s) || y_s > 0x1p126) { return quant_round_binary64(acc, a_s, b_s, y_s); }
219
+ let product_hi = fma(acc_f, ab_hi, 0.0);
220
+ if (!quant_normal_expansion_range(product_hi)) { return quant_round_binary64(acc, a_s, b_s, y_s); }
221
+ let rounded_acc_tail = fma(acc_f, ab_hi, -product_hi) + fma(acc_f, ab_lo, 0.0);
222
+ let product_tail = fma(acc_tail, ab_hi, rounded_acc_tail) + fma(acc_tail, ab_lo, 0.0);
223
+ // Compare the expansion directly. Renormalizing it adds a cancelling
224
+ // subtraction/addition chain that can erase the residual during lowering.
225
+ let denominator_bits = bitcast<u32>(y_s);
226
+ let denominator_power_two = (denominator_bits & 0x7fffffu) == 0u;
227
+ let dyadic_candidate = denominator_power_two && ab_lo == 0.0;
228
+ return round_expansion_ratio(product_hi, product_tail, y_s, ab_hi, dyadic_candidate, acc, a_s, b_s, y_s);
229
+ }
230
+
231
+ // A conservative binary32 error filter accepts only a result safely inside one
232
+ // integer rounding bin. Ambiguous boundaries keep the full expansion/binary64
233
+ // path; this is an exact-result shortcut, not an approximate quantization mode.
234
+ fn round_requantized(acc: i32, a_s: f32, b_s: f32, y_s: f32) -> i32 {
235
+ if (acc == 0) { return 0; }
236
+ let a_bits = bitcast<u32>(a_s);
237
+ let b_bits = bitcast<u32>(b_s);
238
+ let y_bits = bitcast<u32>(y_s);
239
+ let ea = (a_bits >> 23u) & 255u;
240
+ let eb = (b_bits >> 23u) & 255u;
241
+ let ey = (y_bits >> 23u) & 255u;
242
+ // Certify the range before arithmetic: overflow need not produce infinity.
243
+ // Any nonzero i32 converts to a float with biased exponent <=158. These
244
+ // scale-only bounds cover every accumulator and can be shared across lanes.
245
+ // Reserve two significands plus carry bits above FTZ, and a factor of two
246
+ // below overflow for products, division and the expansion midpoint product.
247
+ if (((a_bits | b_bits | y_bits) & 0x80000000u) != 0u ||
248
+ min(ea, eb) == 0u || max(ea, eb) >= 255u ||
249
+ ea + eb < 180u || ea + eb > 347u ||
250
+ ey < 53u || ey > 242u || ea + eb > ey + 220u) {
251
+ return quant_round_binary64(acc, a_s, b_s, y_s);
252
+ }
253
+ let scale = fma(a_s, b_s, 0.0);
254
+ let product = fma(f32(acc), scale, 0.0);
255
+ let estimate = product / y_s;
256
+ if (estimate <= -1024.0) { return -1024; }
257
+ if (estimate >= 1024.0) { return 1024; }
258
+ let nearest = round(estimate);
259
+ // 32 binary32 unit roundoffs reserve room for the i32 conversion, two
260
+ // products, quotient, and distance arithmetic, plus binary64 rounding.
261
+ let margin = max(1.0, abs(estimate)) * 0x1p-19;
262
+ if (abs(estimate - nearest) < 0.5 - margin) { return i32(nearest); }
263
+ {% if device.adapterInfo.vendor != "apple" %}
264
+ // A rounded half-integer estimate can hide which side the exact f32-scale
265
+ // ratio lies on. Resolve that one-ulp neighborhood conservatively outside
266
+ // the compact fast path; exact power-of-two ratios already have a cheap
267
+ // and correct path in round_requantized_precise.
268
+ if (abs(abs(estimate - nearest) - 0.5) <= max(1.0, abs(estimate)) * 0x1p-24) {
269
+ let dyadic_scales = ((a_bits | b_bits | y_bits) & 0x807fffffu) == 0u &&
270
+ min(a_bits, min(b_bits, y_bits)) != 0u &&
271
+ max(a_bits, max(b_bits, y_bits)) < 0x7f800000u;
272
+ if (!dyadic_scales) { return quant_round_binary64(acc, a_s, b_s, y_s); }
273
+ }
274
+ {% endif %}
275
+ return round_requantized_precise(acc, a_s, b_s, y_s);
276
+ }
277
 
278
  const IN_C: u32 = {{ inChannels }}u;
279
  const IN_H: u32 = {{ inHeight }}u;
 
323
  + signed_byte(v, 16u) + signed_byte(v, 24u);
324
  }
325
 
326
+ {{ quant_read_column_zero("read_w_zero", "w_zero_point", false, per_column=wZeroPerChannel, param_name="oc") }}
 
 
 
 
 
 
327
 
328
  fn read_w_scale({% if wScalePerChannel %}oc: u32{% endif %}) -> f32 {
329
  {% if wScalePerChannel %}
 
334
  }
335
 
336
  fn requantize(acc: i32{% if wScalePerChannel %}, oc: u32{% endif %}) -> u32 {
 
337
  let q = clamp(
338
+ round_requantized(acc, x_scale[0], read_w_scale({% if wScalePerChannel %}oc{% endif %}), y_scale[0]) + i32(y_zero_point[0]),
339
  0,
340
  255,
341
  );
build/webgpu/qlinear-conv-nchw-x4.wgsl.jinja CHANGED
@@ -1,46 +1,275 @@
1
- {% macro flat_index_2d(name="i", bound="params.count", guardInline=false, note="dispatch-limit") %}
2
- {% if note == "dispatch-limit" %}
3
- // 2D-folded flat index: gid.y carries the high bits past the dispatch's
4
- // per-axis workgroup fold width (outputs > 16.7M elements).
5
- {% elif note == "limit" %}
6
- // 2D-folded flat index: gid.y carries the high bits past the dispatch's
7
- // per-axis workgroup fold width.
8
- {% elif note == "device-axis" %}
9
- // The flat dispatch is folded across x/y at a fixed per-axis workgroup
10
- // width; gid.y carries the high portion of the output index.
11
- {% elif note == "vec4-limit" %}
12
- // 2D-folded flat vec4 index: gid.y carries the high bits past the dispatch's
13
- // per-axis workgroup fold width (the dispatch caps x and spills into y).
14
- {% elif note == "element-limit" %}
15
- // 2D-folded flat element index: gid.y carries the high bits past the
16
- // dispatch's per-axis workgroup fold width.
17
- {% elif note == "dispatch" %}
18
  // 2D-folded flat index: gid.y carries the high bits past the dispatch's
19
  // per-axis workgroup fold width.
20
- {% endif %}
21
- {% if bound == "" %}
22
- let {{ name }} = gid.x + gid.y * {{ DISPATCH_FOLD_WIDTH }}u * {{ tunables.WORKGROUP_SIZE }}u;
23
- {%- elif guardInline %}
24
- let {{ name }} = gid.x + gid.y * {{ DISPATCH_FOLD_WIDTH }}u * {{ tunables.WORKGROUP_SIZE }}u;
25
- if ({{ name }} >= {{ bound }}) { return; }
26
- {%- else %}
27
- let {{ name }} = gid.x + gid.y * {{ DISPATCH_FOLD_WIDTH }}u * {{ tunables.WORKGROUP_SIZE }}u;
28
- if ({{ name }} >= {{ bound }}) {
29
- return;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
30
  }
31
- {%- endif %}
32
- {% endmacro %}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
33
 
34
- {{ env.wgsl.resourceDeclarations }}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
35
 
36
- fn x_zero() -> i32 {
37
- {% if xUnsigned %}
38
- return i32(x_zero_point[0]);
39
- {% else %}
40
- return x_zero_point[0];
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
41
  {% endif %}
 
42
  }
43
 
 
 
44
  fn w_zero({% if wZeroPerChannel %}oc: u32{% endif %}) -> i32 {
45
  {% if wZeroPerChannel %}
46
  let i = oc;
@@ -70,13 +299,7 @@ fn centered_w(i: u32, zp: i32) -> i32 {
70
  {% endif %}
71
  }
72
 
73
- fn y_zero() -> i32 {
74
- {% if yUnsigned %}
75
- return i32(y_zero_point[0]);
76
- {% else %}
77
- return y_zero_point[0];
78
- {% endif %}
79
- }
80
 
81
  fn requant(acc: i32{% if wScalePerChannel %}, oc: u32{% endif %}) -> i32 {
82
  {% if wScalePerChannel %}
@@ -84,8 +307,7 @@ fn requant(acc: i32{% if wScalePerChannel %}, oc: u32{% endif %}) -> i32 {
84
  {% else %}
85
  let ws = w_scale[0];
86
  {% endif %}
87
- let scaled = f32(acc) * x_scale[0] * ws / y_scale[0];
88
- return clamp(i32(clamp(round(scaled), -2.0e9, 2.0e9)) + y_zero(), {{ qMin }}, {{ qMax }});
89
  }
90
 
91
  fn write_y(i: u32, value: i32) {
@@ -98,7 +320,7 @@ fn write_y(i: u32, value: i32) {
98
 
99
  @compute @workgroup_size({{ tunables.WORKGROUP_SIZE }})
100
  fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
101
- {{ flat_index_2d("flat", "", note="") }}
102
  let groups_per_row = (params.outW + 3u) / 4u;
103
  let oc_blocks = (params.outChannels + {{ ocTile - 1 }}u) / {{ ocTile }}u;
104
  let group_count = params.batch * oc_blocks * params.outH * groups_per_row;
 
1
+ {% macro quant_read_scalar_zero(fn_name, buffer_name, is_unsigned, present=true) %}
2
+ fn {{ fn_name }}() -> i32 {
3
+ {% if is_unsigned %}
4
+ return i32({{ buffer_name }}[0]);
5
+ {% else %}
6
+ return {{ buffer_name }}[0];
7
+ {% endif %}
8
+ }{% endmacro %}
9
+ {% macro flat_index_2d(workgroupSize, name="i", bound="params.count", guardInline=false) %}
10
+ {% set wgTerm = workgroupSize ~ "u" if workgroupSize is number else workgroupSize %}
 
 
 
 
 
 
 
11
  // 2D-folded flat index: gid.y carries the high bits past the dispatch's
12
  // per-axis workgroup fold width.
13
+ let {{ name }} = gid.x + gid.y * {{ DISPATCH_FOLD_WIDTH }}u * {{ wgTerm }};{% endmacro %}
14
+ {{ env.wgsl.resourceDeclarations }}
15
+ // Rare-boundary fallback: reproduce the reference's binary64 scale division
16
+ // and accumulator product using integer significands; no f64 feature required.
17
+ fn quant_u96_add(a: vec3<u32>, b: vec3<u32>) -> vec3<u32> {
18
+ let low = a.x + b.x;
19
+ let middle = a.y + b.y;
20
+ let with_carry = middle + u32(low < a.x);
21
+ return vec3<u32>(low, with_carry, a.z + b.z + u32(middle < a.y) + u32(with_carry < middle));
22
+ }
23
+ fn quant_u96_shr(a: vec3<u32>) -> vec3<u32> {
24
+ return vec3<u32>((a.x >> 1u) | (a.y << 31u), (a.y >> 1u) | (a.z << 31u), a.z >> 1u);
25
+ }
26
+ fn quant_u96_round_shift(value: vec3<u32>, shift: u32) -> vec3<u32> {
27
+ var q = value;
28
+ var sticky = false;
29
+ var guard = false;
30
+ for (var bit = 0u; bit < shift; bit++) {
31
+ sticky = sticky || guard;
32
+ guard = (q.x & 1u) != 0u;
33
+ q = quant_u96_shr(q);
34
+ }
35
+ if (guard && (sticky || (q.x & 1u) != 0u)) {
36
+ q = quant_u96_add(q, vec3<u32>(1u, 0u, 0u));
37
  }
38
+ return q;
39
+ }
40
+ fn quant_float_significand(bits: u32) -> u32 {
41
+ return (bits & 0x7fffffu) | select(0x800000u, 0u, (bits & 0x7f800000u) == 0u);
42
+ }
43
+ fn quant_float_exponent(bits: u32) -> i32 {
44
+ return max(1, i32((bits >> 23u) & 255u)) - 150;
45
+ }
46
+ fn quant_round_binary64(acc: i32, a_s: f32, b_s: f32, y_s: f32) -> i32 {
47
+ if (acc == 0) { return 0; }
48
+ let a_bits = bitcast<u32>(a_s);
49
+ let b_bits = bitcast<u32>(b_s);
50
+ let y_bits = bitcast<u32>(y_s);
51
+ let a = quant_float_significand(a_bits);
52
+ let b = quant_float_significand(b_bits);
53
+ let denominator = quant_float_significand(y_bits);
54
+ if (a == 0u || b == 0u) { return 0; }
55
+ // The exact product of two 24-bit significands occupies at most 48 bits.
56
+ let p0 = (a & 65535u) * (b & 65535u);
57
+ let cross = (a >> 16u) * (b & 65535u) + (b >> 16u) * (a & 65535u);
58
+ let product_low = p0 + (cross << 16u);
59
+ let product_high = (a >> 16u) * (b >> 16u) + (cross >> 16u) + u32(product_low < p0);
60
+ var input_bit = select(31 - i32(countLeadingZeros(product_low)), 63 - i32(countLeadingZeros(product_high)), product_high != 0u);
61
+ var remainder = 0u;
62
+ var significant_bits = 0u;
63
+ var leading_exponent = 0;
64
+ var significand = vec2<u32>(0u);
65
+ // Long division generates the 53 bits of the binary64 multiplier. The
66
+ // remainder fits in 25 bits, including its left shift. At most 24 leading
67
+ // zero bits precede the first significant bit.
68
+ loop {
69
+ var incoming = 0u;
70
+ if (input_bit >= 32) { incoming = (product_high >> u32(input_bit - 32)) & 1u; }
71
+ else if (input_bit >= 0) { incoming = (product_low >> u32(input_bit)) & 1u; }
72
+ remainder = (remainder << 1u) | incoming;
73
+ let digit = u32(remainder >= denominator);
74
+ if (digit != 0u) { remainder -= denominator; }
75
+ if (significant_bits != 0u || digit != 0u) {
76
+ if (significant_bits == 0u) { leading_exponent = input_bit; }
77
+ significand = vec2<u32>((significand.x << 1u) | digit, (significand.y << 1u) | (significand.x >> 31u));
78
+ significant_bits++;
79
+ }
80
+ input_bit--;
81
+ if (significant_bits == 53u) { break; }
82
+ }
83
+ let twice_remainder = remainder << 1u;
84
+ if (twice_remainder > denominator || (twice_remainder == denominator && (significand.x & 1u) != 0u)) {
85
+ let low = significand.x + 1u;
86
+ significand = vec2<u32>(low, significand.y + u32(low == 0u));
87
+ }
88
+ let scale_exponent = quant_float_exponent(a_bits) + quant_float_exponent(b_bits) - quant_float_exponent(y_bits) + leading_exponent - 52;
89
+ // abs(INT_MIN) is representable as u32. Multiply the rounded multiplier's
90
+ // significand by the exact accumulator into three little-endian u32 limbs.
91
+ var magnitude = select(u32(acc), 0u - u32(acc), acc < 0);
92
+ var operand = vec3<u32>(significand, 0u);
93
+ var product = vec3<u32>(0u);
94
+ loop {
95
+ if ((magnitude & 1u) != 0u) { product = quant_u96_add(product, operand); }
96
+ magnitude >>= 1u;
97
+ if (magnitude == 0u) { break; }
98
+ operand = vec3<u32>(operand.x << 1u, (operand.y << 1u) | (operand.x >> 31u), (operand.z << 1u) | (operand.y >> 31u));
99
+ }
100
+ var product_top = 31 - i32(countLeadingZeros(product.x));
101
+ if (product.y != 0u) { product_top = 63 - i32(countLeadingZeros(product.y)); }
102
+ if (product.z != 0u) { product_top = 95 - i32(countLeadingZeros(product.z)); }
103
+ if (product_top + scale_exponent >= 10) { return select(1024, -1024, acc < 0); }
104
+ if (product_top + scale_exponent < -2) { return 0; }
105
+ let product_shift = u32(max(0, product_top - 52));
106
+ let rounded_product = quant_u96_round_shift(product, product_shift);
107
+ let result_exponent = scale_exponent + i32(product_shift);
108
+ var rounded = rounded_product;
109
+ if (result_exponent < 0) { rounded = quant_u96_round_shift(rounded_product, u32(-result_exponent)); }
110
+ else { rounded.x <<= u32(result_exponent); }
111
+ return select(i32(rounded.x), -i32(rounded.x), acc < 0);
112
+ }
113
 
114
+ // A power-of-two division and an exactly represented <=48-bit product need
115
+ // no binary64 boundary resolver. This includes ordinary dyadic quantization.
116
+ fn quant_exact_dyadic_product(acc: i32, scale: f32) -> bool {
117
+ let magnitude = select(u32(acc), 0u - u32(acc), acc < 0);
118
+ let significand = quant_float_significand(bitcast<u32>(scale));
119
+ let integer_bits = 32u - countLeadingZeros(magnitude) - countTrailingZeros(magnitude);
120
+ let scale_bits = 32u - countLeadingZeros(significand) - countTrailingZeros(significand);
121
+ return integer_bits + scale_bits <= 48u;
122
+ }
123
+ fn quant_normal_expansion_range(v: f32) -> bool {
124
+ let exponent = (bitcast<u32>(v) >> 23u) & 255u;
125
+ // Reserve two binary32 significands plus four carry/error bits above the
126
+ // minimum normal exponent, so residual arithmetic avoids FTZ underflow.
127
+ return exponent >= (1u + 2u * 24u + 4u) && exponent < 255u;
128
+ }
129
 
130
+ fn round_expansion_ratio(numerator_hi: f32, numerator_lo: f32, denominator: f32, dyadic_scale: f32, dyadic_candidate: bool, acc: i32, a_s: f32, b_s: f32, y_s: f32) -> i32 {
131
+ let estimate = numerator_hi / denominator;
132
+ // Anything outside this range saturates after adding an 8-bit zero point;
133
+ // returning early also keeps floor/i32 conversion away from Inf and overflow.
134
+ if (estimate <= -1024.0) { return -1024; }
135
+ if (estimate >= 1024.0) { return 1024; }
136
+
137
+ let lo = floor(estimate);
138
+ let hi = lo + 1.0;
139
+ let midpoint = lo + 0.5;
140
+ let midpoint_hi = fma(midpoint, denominator, 0.0);
141
+ if ((bitcast<u32>(midpoint_hi) & 0x7f800000u) == 0x7f800000u) { return quant_round_binary64(acc, a_s, b_s, y_s); }
142
+ let midpoint_lo = fma(midpoint, denominator, -midpoint_hi);
143
+ // The high-word subtraction is exact for these similarly sized products
144
+ // (Sterbenz); adding the residual difference therefore preserves its sign
145
+ // even when the rounded high words straddle adjacent f32 values.
146
+ let midpoint_delta = fma(1.0, numerator_hi - midpoint_hi, numerator_lo - midpoint_lo);
147
+ // Escalate ambiguous expansion signs instead of changing the tie tolerance.
148
+ // Six FMA/product tails and their additions fit within this conservative
149
+ // 16*u^2 error envelope (u=2^-24), including binary64 reference rounding.
150
+ if ((bitcast<u32>(midpoint_delta) & 0x7f800000u) == 0x7f800000u) { return quant_round_binary64(acc, a_s, b_s, y_s); }
151
+ if (abs(midpoint_delta) <= max(abs(numerator_hi), abs(midpoint_hi)) * 0x1p-44) {
152
+ if (!dyadic_candidate || !quant_exact_dyadic_product(acc, dyadic_scale)) { return quant_round_binary64(acc, a_s, b_s, y_s); }
153
+ }
154
+ if (midpoint_delta < 0.0) {
155
+ return i32(lo);
156
+ }
157
+ if (midpoint_delta > 0.0) {
158
+ return i32(hi);
159
+ }
160
+ let half_lo = floor(lo * 0.5);
161
+ let lo_is_even = lo - half_lo * 2.0 == 0.0;
162
+ return i32(select(hi, lo, lo_is_even));
163
+ }
164
+
165
+ // Multiplication by a power of two keeps every i32 bit in binary64. Round the
166
+ // exact dyadic value directly, including INT_MIN, half-even ties and saturation.
167
+ fn quant_round_power_two(acc: i32, exponent: i32) -> i32 {
168
+ let magnitude = select(u32(acc), 0u - u32(acc), acc < 0);
169
+ var rounded = 0u;
170
+ if (exponent >= 0) {
171
+ if (exponent >= 10) { return select(1024, -1024, acc < 0); }
172
+ rounded = min(magnitude, 1024u >> u32(exponent)) << u32(exponent);
173
+ } else {
174
+ let shift = u32(-exponent);
175
+ if (shift >= 32u) { return 0; }
176
+ let whole = magnitude >> shift;
177
+ let remainder = magnitude & ((1u << shift) - 1u);
178
+ let half = 1u << (shift - 1u);
179
+ rounded = min(1024u, whole + u32(remainder > half || (remainder == half && (whole & 1u) != 0u)));
180
+ }
181
+ return select(i32(rounded), -i32(rounded), acc < 0);
182
+ }
183
+
184
+ fn round_requantized_precise(acc: i32, a_s: f32, b_s: f32, y_s: f32) -> i32 {
185
+ if (acc == 0) { return 0; }
186
+ let a_bits = bitcast<u32>(a_s);
187
+ let b_bits = bitcast<u32>(b_s);
188
+ let y_bits = bitcast<u32>(y_s);
189
+ if (((a_bits | b_bits | y_bits) & 0x807fffffu) == 0u && min(a_bits, min(b_bits, y_bits)) != 0u && max(a_bits, max(b_bits, y_bits)) < 0x7f800000u) {
190
+ let exponent = i32(a_bits >> 23u) + i32(b_bits >> 23u) - i32(y_bits >> 23u) - 127;
191
+ return quant_round_power_two(acc, exponent);
192
+ }
193
+ let acc_f = f32(acc);
194
+ // Every i32 high part aligned to 256 is exactly representable in f32.
195
+ // Recover the conversion residual without an out-of-range i32(acc_f)
196
+ // conversion when INT_MAX rounds up to 2^31.
197
+ let acc_tail = fma(-1.0, acc_f - f32(acc & -256), f32(acc & 255));
198
+ // Equal b/y scales cancel exactly. Bypass their product and division so a
199
+ // rounded intermediate cannot move a near-half value onto the tie.
200
+ if (bitcast<u32>(b_s) == bitcast<u32>(y_s)) {
201
+ let direct_hi = fma(acc_f, a_s, 0.0);
202
+ let direct_lo = fma(acc_tail, a_s, fma(acc_f, a_s, -direct_hi));
203
+ if (!quant_normal_expansion_range(direct_hi)) { return quant_round_binary64(acc, a_s, b_s, y_s); }
204
+ return round_expansion_ratio(direct_hi, direct_lo, 1.0, a_s, true, acc, a_s, b_s, y_s);
205
+ }
206
+
207
+ // Keep the scale product as a two-float expansion. Its FMA residual preserves
208
+ // which side of a rounding midpoint a near-half value occupies.
209
+ let ab_hi = fma(a_s, b_s, 0.0);
210
+ let ab_lo = fma(a_s, b_s, -ab_hi);
211
+ if (!quant_normal_expansion_range(ab_hi) || !quant_normal_expansion_range(y_s) || y_s > 0x1p126) { return quant_round_binary64(acc, a_s, b_s, y_s); }
212
+ let product_hi = fma(acc_f, ab_hi, 0.0);
213
+ if (!quant_normal_expansion_range(product_hi)) { return quant_round_binary64(acc, a_s, b_s, y_s); }
214
+ let rounded_acc_tail = fma(acc_f, ab_hi, -product_hi) + fma(acc_f, ab_lo, 0.0);
215
+ let product_tail = fma(acc_tail, ab_hi, rounded_acc_tail) + fma(acc_tail, ab_lo, 0.0);
216
+ // Compare the expansion directly. Renormalizing it adds a cancelling
217
+ // subtraction/addition chain that can erase the residual during lowering.
218
+ let denominator_bits = bitcast<u32>(y_s);
219
+ let denominator_power_two = (denominator_bits & 0x7fffffu) == 0u;
220
+ let dyadic_candidate = denominator_power_two && ab_lo == 0.0;
221
+ return round_expansion_ratio(product_hi, product_tail, y_s, ab_hi, dyadic_candidate, acc, a_s, b_s, y_s);
222
+ }
223
+
224
+ // A conservative binary32 error filter accepts only a result safely inside one
225
+ // integer rounding bin. Ambiguous boundaries keep the full expansion/binary64
226
+ // path; this is an exact-result shortcut, not an approximate quantization mode.
227
+ fn round_requantized(acc: i32, a_s: f32, b_s: f32, y_s: f32) -> i32 {
228
+ if (acc == 0) { return 0; }
229
+ let a_bits = bitcast<u32>(a_s);
230
+ let b_bits = bitcast<u32>(b_s);
231
+ let y_bits = bitcast<u32>(y_s);
232
+ let ea = (a_bits >> 23u) & 255u;
233
+ let eb = (b_bits >> 23u) & 255u;
234
+ let ey = (y_bits >> 23u) & 255u;
235
+ // Certify the range before arithmetic: overflow need not produce infinity.
236
+ // Any nonzero i32 converts to a float with biased exponent <=158. These
237
+ // scale-only bounds cover every accumulator and can be shared across lanes.
238
+ // Reserve two significands plus carry bits above FTZ, and a factor of two
239
+ // below overflow for products, division and the expansion midpoint product.
240
+ if (((a_bits | b_bits | y_bits) & 0x80000000u) != 0u ||
241
+ min(ea, eb) == 0u || max(ea, eb) >= 255u ||
242
+ ea + eb < 180u || ea + eb > 347u ||
243
+ ey < 53u || ey > 242u || ea + eb > ey + 220u) {
244
+ return quant_round_binary64(acc, a_s, b_s, y_s);
245
+ }
246
+ let scale = fma(a_s, b_s, 0.0);
247
+ let product = fma(f32(acc), scale, 0.0);
248
+ let estimate = product / y_s;
249
+ if (estimate <= -1024.0) { return -1024; }
250
+ if (estimate >= 1024.0) { return 1024; }
251
+ let nearest = round(estimate);
252
+ // 32 binary32 unit roundoffs reserve room for the i32 conversion, two
253
+ // products, quotient, and distance arithmetic, plus binary64 rounding.
254
+ let margin = max(1.0, abs(estimate)) * 0x1p-19;
255
+ if (abs(estimate - nearest) < 0.5 - margin) { return i32(nearest); }
256
+ {% if device.adapterInfo.vendor != "apple" %}
257
+ // A rounded half-integer estimate can hide which side the exact f32-scale
258
+ // ratio lies on. Resolve that one-ulp neighborhood conservatively outside
259
+ // the compact fast path; exact power-of-two ratios already have a cheap
260
+ // and correct path in round_requantized_precise.
261
+ if (abs(abs(estimate - nearest) - 0.5) <= max(1.0, abs(estimate)) * 0x1p-24) {
262
+ let dyadic_scales = ((a_bits | b_bits | y_bits) & 0x807fffffu) == 0u &&
263
+ min(a_bits, min(b_bits, y_bits)) != 0u &&
264
+ max(a_bits, max(b_bits, y_bits)) < 0x7f800000u;
265
+ if (!dyadic_scales) { return quant_round_binary64(acc, a_s, b_s, y_s); }
266
+ }
267
  {% endif %}
268
+ return round_requantized_precise(acc, a_s, b_s, y_s);
269
  }
270
 
271
+ {{ quant_read_scalar_zero("x_zero", "x_zero_point", xUnsigned) }}
272
+
273
  fn w_zero({% if wZeroPerChannel %}oc: u32{% endif %}) -> i32 {
274
  {% if wZeroPerChannel %}
275
  let i = oc;
 
299
  {% endif %}
300
  }
301
 
302
+ {{ quant_read_scalar_zero("y_zero", "y_zero_point", yUnsigned) }}
 
 
 
 
 
 
303
 
304
  fn requant(acc: i32{% if wScalePerChannel %}, oc: u32{% endif %}) -> i32 {
305
  {% if wScalePerChannel %}
 
307
  {% else %}
308
  let ws = w_scale[0];
309
  {% endif %}
310
+ return clamp(round_requantized(acc, x_scale[0], ws, y_scale[0]) + y_zero(), {{ qMin }}, {{ qMax }});
 
311
  }
312
 
313
  fn write_y(i: u32, value: i32) {
 
320
 
321
  @compute @workgroup_size({{ tunables.WORKGROUP_SIZE }})
322
  fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
323
+ {{ flat_index_2d(tunables.WORKGROUP_SIZE, "flat", "") }}
324
  let groups_per_row = (params.outW + 3u) / 4u;
325
  let oc_blocks = (params.outChannels + {{ ocTile - 1 }}u) / {{ ocTile }}u;
326
  let group_count = params.batch * oc_blocks * params.outH * groups_per_row;
build/webgpu/qlinear-conv-requantize.wgsl.jinja CHANGED
@@ -1,46 +1,276 @@
1
- {% macro flat_index_2d(name="i", bound="params.count", guardInline=false, note="dispatch-limit") %}
2
- {% if note == "dispatch-limit" %}
3
- // 2D-folded flat index: gid.y carries the high bits past the dispatch's
4
- // per-axis workgroup fold width (outputs > 16.7M elements).
5
- {% elif note == "limit" %}
6
- // 2D-folded flat index: gid.y carries the high bits past the dispatch's
7
- // per-axis workgroup fold width.
8
- {% elif note == "device-axis" %}
9
- // The flat dispatch is folded across x/y at a fixed per-axis workgroup
10
- // width; gid.y carries the high portion of the output index.
11
- {% elif note == "vec4-limit" %}
12
- // 2D-folded flat vec4 index: gid.y carries the high bits past the dispatch's
13
- // per-axis workgroup fold width (the dispatch caps x and spills into y).
14
- {% elif note == "element-limit" %}
15
- // 2D-folded flat element index: gid.y carries the high bits past the
16
- // dispatch's per-axis workgroup fold width.
17
- {% elif note == "dispatch" %}
18
  // 2D-folded flat index: gid.y carries the high bits past the dispatch's
19
  // per-axis workgroup fold width.
20
- {% endif %}
21
- {% if bound == "" %}
22
- let {{ name }} = gid.x + gid.y * {{ DISPATCH_FOLD_WIDTH }}u * {{ tunables.WORKGROUP_SIZE }}u;
23
- {%- elif guardInline %}
24
- let {{ name }} = gid.x + gid.y * {{ DISPATCH_FOLD_WIDTH }}u * {{ tunables.WORKGROUP_SIZE }}u;
25
- if ({{ name }} >= {{ bound }}) { return; }
26
- {%- else %}
27
- let {{ name }} = gid.x + gid.y * {{ DISPATCH_FOLD_WIDTH }}u * {{ tunables.WORKGROUP_SIZE }}u;
28
- if ({{ name }} >= {{ bound }}) {
29
- return;
 
 
 
 
 
 
 
 
 
 
 
 
30
  }
31
- {%- endif %}
32
- {% endmacro %}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
33
 
34
- {{ env.wgsl.resourceDeclarations }}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
35
 
36
- fn read_y_zero() -> i32 {
37
- {% if yUnsigned %}
38
- return i32(y_zero_point[0]);
39
- {% else %}
40
- return y_zero_point[0];
41
- {% endif %}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
42
  }
43
 
 
 
44
  fn write_y(index: u32, value: i32) {
45
  {% if yUnsigned %}
46
  y[index] = u32(value);
@@ -51,7 +281,7 @@ fn write_y(index: u32, value: i32) {
51
 
52
  @compute @workgroup_size({{ tunables.WORKGROUP_SIZE }})
53
  fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
54
- {{ flat_index_2d(guardInline=true) }}
55
  {% if hasBias or wScalePerChannel %}
56
  let oc = (i / params.outImageSize) % params.outChannels;
57
  {% endif %}
@@ -67,10 +297,6 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
67
  {% else %}
68
  let w_scale_value = w_scale[0];
69
  {% endif %}
70
- let scaled = f32(raw) * x_scale[0] * w_scale_value / y_scale[0];
71
- // Clamp the rounded float into an i32-representable range before the cast so a
72
- // huge-finite / Inf requantized value saturates (the final clamp pins it to
73
- // [qMin,qMax]) instead of invoking undefined behaviour in i32(huge_float).
74
- let q = clamp(i32(clamp(round(scaled), -2.0e9, 2.0e9)) + read_y_zero(), {{ qMin }}, {{ qMax }});
75
  write_y(i, q);
76
  }
 
1
+ {% macro quant_read_scalar_zero(fn_name, buffer_name, is_unsigned, present=true) %}
2
+ fn {{ fn_name }}() -> i32 {
3
+ {% if is_unsigned %}
4
+ return i32({{ buffer_name }}[0]);
5
+ {% else %}
6
+ return {{ buffer_name }}[0];
7
+ {% endif %}
8
+ }{% endmacro %}
9
+ {% macro flat_index_2d(workgroupSize, name="i", bound="params.count", guardInline=false) %}
10
+ {% set wgTerm = workgroupSize ~ "u" if workgroupSize is number else workgroupSize %}
 
 
 
 
 
 
 
11
  // 2D-folded flat index: gid.y carries the high bits past the dispatch's
12
  // per-axis workgroup fold width.
13
+ let {{ name }} = gid.x + gid.y * {{ DISPATCH_FOLD_WIDTH }}u * {{ wgTerm }};
14
+ if ({{ name }} >= {{ bound }}) { return; }{% endmacro %}
15
+ {{ env.wgsl.resourceDeclarations }}
16
+ // Rare-boundary fallback: reproduce the reference's binary64 scale division
17
+ // and accumulator product using integer significands; no f64 feature required.
18
+ fn quant_u96_add(a: vec3<u32>, b: vec3<u32>) -> vec3<u32> {
19
+ let low = a.x + b.x;
20
+ let middle = a.y + b.y;
21
+ let with_carry = middle + u32(low < a.x);
22
+ return vec3<u32>(low, with_carry, a.z + b.z + u32(middle < a.y) + u32(with_carry < middle));
23
+ }
24
+ fn quant_u96_shr(a: vec3<u32>) -> vec3<u32> {
25
+ return vec3<u32>((a.x >> 1u) | (a.y << 31u), (a.y >> 1u) | (a.z << 31u), a.z >> 1u);
26
+ }
27
+ fn quant_u96_round_shift(value: vec3<u32>, shift: u32) -> vec3<u32> {
28
+ var q = value;
29
+ var sticky = false;
30
+ var guard = false;
31
+ for (var bit = 0u; bit < shift; bit++) {
32
+ sticky = sticky || guard;
33
+ guard = (q.x & 1u) != 0u;
34
+ q = quant_u96_shr(q);
35
  }
36
+ if (guard && (sticky || (q.x & 1u) != 0u)) {
37
+ q = quant_u96_add(q, vec3<u32>(1u, 0u, 0u));
38
+ }
39
+ return q;
40
+ }
41
+ fn quant_float_significand(bits: u32) -> u32 {
42
+ return (bits & 0x7fffffu) | select(0x800000u, 0u, (bits & 0x7f800000u) == 0u);
43
+ }
44
+ fn quant_float_exponent(bits: u32) -> i32 {
45
+ return max(1, i32((bits >> 23u) & 255u)) - 150;
46
+ }
47
+ fn quant_round_binary64(acc: i32, a_s: f32, b_s: f32, y_s: f32) -> i32 {
48
+ if (acc == 0) { return 0; }
49
+ let a_bits = bitcast<u32>(a_s);
50
+ let b_bits = bitcast<u32>(b_s);
51
+ let y_bits = bitcast<u32>(y_s);
52
+ let a = quant_float_significand(a_bits);
53
+ let b = quant_float_significand(b_bits);
54
+ let denominator = quant_float_significand(y_bits);
55
+ if (a == 0u || b == 0u) { return 0; }
56
+ // The exact product of two 24-bit significands occupies at most 48 bits.
57
+ let p0 = (a & 65535u) * (b & 65535u);
58
+ let cross = (a >> 16u) * (b & 65535u) + (b >> 16u) * (a & 65535u);
59
+ let product_low = p0 + (cross << 16u);
60
+ let product_high = (a >> 16u) * (b >> 16u) + (cross >> 16u) + u32(product_low < p0);
61
+ var input_bit = select(31 - i32(countLeadingZeros(product_low)), 63 - i32(countLeadingZeros(product_high)), product_high != 0u);
62
+ var remainder = 0u;
63
+ var significant_bits = 0u;
64
+ var leading_exponent = 0;
65
+ var significand = vec2<u32>(0u);
66
+ // Long division generates the 53 bits of the binary64 multiplier. The
67
+ // remainder fits in 25 bits, including its left shift. At most 24 leading
68
+ // zero bits precede the first significant bit.
69
+ loop {
70
+ var incoming = 0u;
71
+ if (input_bit >= 32) { incoming = (product_high >> u32(input_bit - 32)) & 1u; }
72
+ else if (input_bit >= 0) { incoming = (product_low >> u32(input_bit)) & 1u; }
73
+ remainder = (remainder << 1u) | incoming;
74
+ let digit = u32(remainder >= denominator);
75
+ if (digit != 0u) { remainder -= denominator; }
76
+ if (significant_bits != 0u || digit != 0u) {
77
+ if (significant_bits == 0u) { leading_exponent = input_bit; }
78
+ significand = vec2<u32>((significand.x << 1u) | digit, (significand.y << 1u) | (significand.x >> 31u));
79
+ significant_bits++;
80
+ }
81
+ input_bit--;
82
+ if (significant_bits == 53u) { break; }
83
+ }
84
+ let twice_remainder = remainder << 1u;
85
+ if (twice_remainder > denominator || (twice_remainder == denominator && (significand.x & 1u) != 0u)) {
86
+ let low = significand.x + 1u;
87
+ significand = vec2<u32>(low, significand.y + u32(low == 0u));
88
+ }
89
+ let scale_exponent = quant_float_exponent(a_bits) + quant_float_exponent(b_bits) - quant_float_exponent(y_bits) + leading_exponent - 52;
90
+ // abs(INT_MIN) is representable as u32. Multiply the rounded multiplier's
91
+ // significand by the exact accumulator into three little-endian u32 limbs.
92
+ var magnitude = select(u32(acc), 0u - u32(acc), acc < 0);
93
+ var operand = vec3<u32>(significand, 0u);
94
+ var product = vec3<u32>(0u);
95
+ loop {
96
+ if ((magnitude & 1u) != 0u) { product = quant_u96_add(product, operand); }
97
+ magnitude >>= 1u;
98
+ if (magnitude == 0u) { break; }
99
+ operand = vec3<u32>(operand.x << 1u, (operand.y << 1u) | (operand.x >> 31u), (operand.z << 1u) | (operand.y >> 31u));
100
+ }
101
+ var product_top = 31 - i32(countLeadingZeros(product.x));
102
+ if (product.y != 0u) { product_top = 63 - i32(countLeadingZeros(product.y)); }
103
+ if (product.z != 0u) { product_top = 95 - i32(countLeadingZeros(product.z)); }
104
+ if (product_top + scale_exponent >= 10) { return select(1024, -1024, acc < 0); }
105
+ if (product_top + scale_exponent < -2) { return 0; }
106
+ let product_shift = u32(max(0, product_top - 52));
107
+ let rounded_product = quant_u96_round_shift(product, product_shift);
108
+ let result_exponent = scale_exponent + i32(product_shift);
109
+ var rounded = rounded_product;
110
+ if (result_exponent < 0) { rounded = quant_u96_round_shift(rounded_product, u32(-result_exponent)); }
111
+ else { rounded.x <<= u32(result_exponent); }
112
+ return select(i32(rounded.x), -i32(rounded.x), acc < 0);
113
+ }
114
 
115
+ // A power-of-two division and an exactly represented <=48-bit product need
116
+ // no binary64 boundary resolver. This includes ordinary dyadic quantization.
117
+ fn quant_exact_dyadic_product(acc: i32, scale: f32) -> bool {
118
+ let magnitude = select(u32(acc), 0u - u32(acc), acc < 0);
119
+ let significand = quant_float_significand(bitcast<u32>(scale));
120
+ let integer_bits = 32u - countLeadingZeros(magnitude) - countTrailingZeros(magnitude);
121
+ let scale_bits = 32u - countLeadingZeros(significand) - countTrailingZeros(significand);
122
+ return integer_bits + scale_bits <= 48u;
123
+ }
124
+ fn quant_normal_expansion_range(v: f32) -> bool {
125
+ let exponent = (bitcast<u32>(v) >> 23u) & 255u;
126
+ // Reserve two binary32 significands plus four carry/error bits above the
127
+ // minimum normal exponent, so residual arithmetic avoids FTZ underflow.
128
+ return exponent >= (1u + 2u * 24u + 4u) && exponent < 255u;
129
+ }
130
 
131
+ fn round_expansion_ratio(numerator_hi: f32, numerator_lo: f32, denominator: f32, dyadic_scale: f32, dyadic_candidate: bool, acc: i32, a_s: f32, b_s: f32, y_s: f32) -> i32 {
132
+ let estimate = numerator_hi / denominator;
133
+ // Anything outside this range saturates after adding an 8-bit zero point;
134
+ // returning early also keeps floor/i32 conversion away from Inf and overflow.
135
+ if (estimate <= -1024.0) { return -1024; }
136
+ if (estimate >= 1024.0) { return 1024; }
137
+
138
+ let lo = floor(estimate);
139
+ let hi = lo + 1.0;
140
+ let midpoint = lo + 0.5;
141
+ let midpoint_hi = fma(midpoint, denominator, 0.0);
142
+ if ((bitcast<u32>(midpoint_hi) & 0x7f800000u) == 0x7f800000u) { return quant_round_binary64(acc, a_s, b_s, y_s); }
143
+ let midpoint_lo = fma(midpoint, denominator, -midpoint_hi);
144
+ // The high-word subtraction is exact for these similarly sized products
145
+ // (Sterbenz); adding the residual difference therefore preserves its sign
146
+ // even when the rounded high words straddle adjacent f32 values.
147
+ let midpoint_delta = fma(1.0, numerator_hi - midpoint_hi, numerator_lo - midpoint_lo);
148
+ // Escalate ambiguous expansion signs instead of changing the tie tolerance.
149
+ // Six FMA/product tails and their additions fit within this conservative
150
+ // 16*u^2 error envelope (u=2^-24), including binary64 reference rounding.
151
+ if ((bitcast<u32>(midpoint_delta) & 0x7f800000u) == 0x7f800000u) { return quant_round_binary64(acc, a_s, b_s, y_s); }
152
+ if (abs(midpoint_delta) <= max(abs(numerator_hi), abs(midpoint_hi)) * 0x1p-44) {
153
+ if (!dyadic_candidate || !quant_exact_dyadic_product(acc, dyadic_scale)) { return quant_round_binary64(acc, a_s, b_s, y_s); }
154
+ }
155
+ if (midpoint_delta < 0.0) {
156
+ return i32(lo);
157
+ }
158
+ if (midpoint_delta > 0.0) {
159
+ return i32(hi);
160
+ }
161
+ let half_lo = floor(lo * 0.5);
162
+ let lo_is_even = lo - half_lo * 2.0 == 0.0;
163
+ return i32(select(hi, lo, lo_is_even));
164
+ }
165
+
166
+ // Multiplication by a power of two keeps every i32 bit in binary64. Round the
167
+ // exact dyadic value directly, including INT_MIN, half-even ties and saturation.
168
+ fn quant_round_power_two(acc: i32, exponent: i32) -> i32 {
169
+ let magnitude = select(u32(acc), 0u - u32(acc), acc < 0);
170
+ var rounded = 0u;
171
+ if (exponent >= 0) {
172
+ if (exponent >= 10) { return select(1024, -1024, acc < 0); }
173
+ rounded = min(magnitude, 1024u >> u32(exponent)) << u32(exponent);
174
+ } else {
175
+ let shift = u32(-exponent);
176
+ if (shift >= 32u) { return 0; }
177
+ let whole = magnitude >> shift;
178
+ let remainder = magnitude & ((1u << shift) - 1u);
179
+ let half = 1u << (shift - 1u);
180
+ rounded = min(1024u, whole + u32(remainder > half || (remainder == half && (whole & 1u) != 0u)));
181
+ }
182
+ return select(i32(rounded), -i32(rounded), acc < 0);
183
+ }
184
+
185
+ fn round_requantized_precise(acc: i32, a_s: f32, b_s: f32, y_s: f32) -> i32 {
186
+ if (acc == 0) { return 0; }
187
+ let a_bits = bitcast<u32>(a_s);
188
+ let b_bits = bitcast<u32>(b_s);
189
+ let y_bits = bitcast<u32>(y_s);
190
+ if (((a_bits | b_bits | y_bits) & 0x807fffffu) == 0u && min(a_bits, min(b_bits, y_bits)) != 0u && max(a_bits, max(b_bits, y_bits)) < 0x7f800000u) {
191
+ let exponent = i32(a_bits >> 23u) + i32(b_bits >> 23u) - i32(y_bits >> 23u) - 127;
192
+ return quant_round_power_two(acc, exponent);
193
+ }
194
+ let acc_f = f32(acc);
195
+ // Every i32 high part aligned to 256 is exactly representable in f32.
196
+ // Recover the conversion residual without an out-of-range i32(acc_f)
197
+ // conversion when INT_MAX rounds up to 2^31.
198
+ let acc_tail = fma(-1.0, acc_f - f32(acc & -256), f32(acc & 255));
199
+ // Equal b/y scales cancel exactly. Bypass their product and division so a
200
+ // rounded intermediate cannot move a near-half value onto the tie.
201
+ if (bitcast<u32>(b_s) == bitcast<u32>(y_s)) {
202
+ let direct_hi = fma(acc_f, a_s, 0.0);
203
+ let direct_lo = fma(acc_tail, a_s, fma(acc_f, a_s, -direct_hi));
204
+ if (!quant_normal_expansion_range(direct_hi)) { return quant_round_binary64(acc, a_s, b_s, y_s); }
205
+ return round_expansion_ratio(direct_hi, direct_lo, 1.0, a_s, true, acc, a_s, b_s, y_s);
206
+ }
207
+
208
+ // Keep the scale product as a two-float expansion. Its FMA residual preserves
209
+ // which side of a rounding midpoint a near-half value occupies.
210
+ let ab_hi = fma(a_s, b_s, 0.0);
211
+ let ab_lo = fma(a_s, b_s, -ab_hi);
212
+ if (!quant_normal_expansion_range(ab_hi) || !quant_normal_expansion_range(y_s) || y_s > 0x1p126) { return quant_round_binary64(acc, a_s, b_s, y_s); }
213
+ let product_hi = fma(acc_f, ab_hi, 0.0);
214
+ if (!quant_normal_expansion_range(product_hi)) { return quant_round_binary64(acc, a_s, b_s, y_s); }
215
+ let rounded_acc_tail = fma(acc_f, ab_hi, -product_hi) + fma(acc_f, ab_lo, 0.0);
216
+ let product_tail = fma(acc_tail, ab_hi, rounded_acc_tail) + fma(acc_tail, ab_lo, 0.0);
217
+ // Compare the expansion directly. Renormalizing it adds a cancelling
218
+ // subtraction/addition chain that can erase the residual during lowering.
219
+ let denominator_bits = bitcast<u32>(y_s);
220
+ let denominator_power_two = (denominator_bits & 0x7fffffu) == 0u;
221
+ let dyadic_candidate = denominator_power_two && ab_lo == 0.0;
222
+ return round_expansion_ratio(product_hi, product_tail, y_s, ab_hi, dyadic_candidate, acc, a_s, b_s, y_s);
223
+ }
224
+
225
+ // A conservative binary32 error filter accepts only a result safely inside one
226
+ // integer rounding bin. Ambiguous boundaries keep the full expansion/binary64
227
+ // path; this is an exact-result shortcut, not an approximate quantization mode.
228
+ fn round_requantized(acc: i32, a_s: f32, b_s: f32, y_s: f32) -> i32 {
229
+ if (acc == 0) { return 0; }
230
+ let a_bits = bitcast<u32>(a_s);
231
+ let b_bits = bitcast<u32>(b_s);
232
+ let y_bits = bitcast<u32>(y_s);
233
+ let ea = (a_bits >> 23u) & 255u;
234
+ let eb = (b_bits >> 23u) & 255u;
235
+ let ey = (y_bits >> 23u) & 255u;
236
+ // Certify the range before arithmetic: overflow need not produce infinity.
237
+ // Any nonzero i32 converts to a float with biased exponent <=158. These
238
+ // scale-only bounds cover every accumulator and can be shared across lanes.
239
+ // Reserve two significands plus carry bits above FTZ, and a factor of two
240
+ // below overflow for products, division and the expansion midpoint product.
241
+ if (((a_bits | b_bits | y_bits) & 0x80000000u) != 0u ||
242
+ min(ea, eb) == 0u || max(ea, eb) >= 255u ||
243
+ ea + eb < 180u || ea + eb > 347u ||
244
+ ey < 53u || ey > 242u || ea + eb > ey + 220u) {
245
+ return quant_round_binary64(acc, a_s, b_s, y_s);
246
+ }
247
+ let scale = fma(a_s, b_s, 0.0);
248
+ let product = fma(f32(acc), scale, 0.0);
249
+ let estimate = product / y_s;
250
+ if (estimate <= -1024.0) { return -1024; }
251
+ if (estimate >= 1024.0) { return 1024; }
252
+ let nearest = round(estimate);
253
+ // 32 binary32 unit roundoffs reserve room for the i32 conversion, two
254
+ // products, quotient, and distance arithmetic, plus binary64 rounding.
255
+ let margin = max(1.0, abs(estimate)) * 0x1p-19;
256
+ if (abs(estimate - nearest) < 0.5 - margin) { return i32(nearest); }
257
+ {% if device.adapterInfo.vendor != "apple" %}
258
+ // A rounded half-integer estimate can hide which side the exact f32-scale
259
+ // ratio lies on. Resolve that one-ulp neighborhood conservatively outside
260
+ // the compact fast path; exact power-of-two ratios already have a cheap
261
+ // and correct path in round_requantized_precise.
262
+ if (abs(abs(estimate - nearest) - 0.5) <= max(1.0, abs(estimate)) * 0x1p-24) {
263
+ let dyadic_scales = ((a_bits | b_bits | y_bits) & 0x807fffffu) == 0u &&
264
+ min(a_bits, min(b_bits, y_bits)) != 0u &&
265
+ max(a_bits, max(b_bits, y_bits)) < 0x7f800000u;
266
+ if (!dyadic_scales) { return quant_round_binary64(acc, a_s, b_s, y_s); }
267
+ }
268
+ {% endif %}
269
+ return round_requantized_precise(acc, a_s, b_s, y_s);
270
  }
271
 
272
+ {{ quant_read_scalar_zero("read_y_zero", "y_zero_point", yUnsigned) }}
273
+
274
  fn write_y(index: u32, value: i32) {
275
  {% if yUnsigned %}
276
  y[index] = u32(value);
 
281
 
282
  @compute @workgroup_size({{ tunables.WORKGROUP_SIZE }})
283
  fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
284
+ {{ flat_index_2d(tunables.WORKGROUP_SIZE, guardInline=true) }}
285
  {% if hasBias or wScalePerChannel %}
286
  let oc = (i / params.outImageSize) % params.outChannels;
287
  {% endif %}
 
297
  {% else %}
298
  let w_scale_value = w_scale[0];
299
  {% endif %}
300
+ let q = clamp(round_requantized(raw, x_scale[0], w_scale_value, y_scale[0]) + read_y_zero(), {{ qMin }}, {{ qMax }});
 
 
 
 
301
  write_y(i, q);
302
  }
build/webgpu/quant-dp4a-matmul.wgsl.jinja CHANGED
@@ -1,6 +1,27 @@
1
- {% set yUnsigned = yUnsigned is defined and yUnsigned %}
2
- {% set qMin = qMin | default(0 if yUnsigned else 0 - 128) %}
3
- {% set qMax = qMax | default(255 if yUnsigned else 127) %}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4
  {% if not env.device.wgslLanguageFeatures.has("packed_4x8_integer_dot_product") %}
5
  // Runtime source-compatible fallbacks for the packed 4x8 integer built-ins used
6
  // by quantized kernels. Native implementations remain unshadowed when the
@@ -21,7 +42,263 @@
21
  }
22
  {% endif %}
23
  {{ env.wgsl.resourceDeclarations }}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
24
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
25
 
26
  const TILE_M: u32 = {{ tileM }}u;
27
  const TILE_N: u32 = {{ tileN }}u;
@@ -64,7 +341,6 @@ fn dot4_packed(a_word: u32, b_word: u32) -> i32 {
64
  return dot4I8Packed(a_word, b_word);
65
  }
66
 
67
-
68
  fn sum4_b(b0: {{ bScalar }}, b1: {{ bScalar }}, b2: {{ bScalar }}, b3: {{ bScalar }}) -> i32 {
69
  {% if bUnsigned %}
70
  return i32(b0 + b1 + b2 + b3) - 512;
@@ -73,38 +349,12 @@ fn sum4_b(b0: {{ bScalar }}, b1: {{ bScalar }}, b2: {{ bScalar }}, b3: {{ bScala
73
  {% endif %}
74
  }
75
 
76
- fn read_a_zero({% if aZeroPerRow %}row: u32{% endif %}) -> i32 {
77
- {% if aZeroPerRow %}
78
- {% if aUnsigned %}
79
- return i32(a_zero_point[row]);
80
- {% else %}
81
- return a_zero_point[row];
82
- {% endif %}
83
- {% else %}
84
- {% if aUnsigned %}
85
- return i32(a_zero_point[0]);
86
- {% else %}
87
- return a_zero_point[0];
88
- {% endif %}
89
- {% endif %}
90
- }
91
 
92
- fn read_b_zero() -> i32 {
93
- {% if bUnsigned %}
94
- return i32(b_zero_point[0]);
95
- {% else %}
96
- return b_zero_point[0];
97
- {% endif %}
98
- }
99
 
100
  {% if requant != "none" %}
101
- fn read_y_zero() -> i32 {
102
- {% if yUnsigned %}
103
- return i32(y_zero_point[0]);
104
- {% else %}
105
- return y_zero_point[0];
106
- {% endif %}
107
- }
108
 
109
  {% endif %}
110
  @compute @workgroup_size(TILE_N, 1, 1)
@@ -252,17 +502,14 @@ fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_id) lid:
252
  acc = acc + ca * sum_b;
253
  acc = acc + kcc;
254
  {% if hasBias %}
255
- acc = acc + bias[mr];
256
  {% endif %}
257
  {% if requant == "none" %}
258
  y[y_base + mr * params.N + col] = acc;
259
  {% else %}
260
  {% set scale_index = "col" if requant == "per_column" else ("mr" if requant == "per_row" else "0u") %}
261
- let scaled = f32(acc) * f32(a_scale[0]) * f32(b_scale[{{ scale_index }}]) / f32(y_scale[0]);
262
- // Clamp the rounded float into an i32-representable range before the cast
263
- // so a huge-finite / Inf requantized value saturates (the final clamp pins
264
- // it to [qMin,qMax]) instead of invoking undefined i32(huge_float).
265
- let q = clamp(i32(clamp(round(scaled), -2.0e9, 2.0e9)) + read_y_zero(), {{ qMin }}, {{ qMax }});
266
  {% if yUnsigned %}
267
  y[y_base + mr * params.N + col] = u32(q);
268
  {% else %}
 
1
+ {% macro quant_read_scalar_zero(fn_name, buffer_name, is_unsigned, present=true) %}
2
+ fn {{ fn_name }}() -> i32 {
3
+ {% if is_unsigned %}
4
+ return i32({{ buffer_name }}[0]);
5
+ {% else %}
6
+ return {{ buffer_name }}[0];
7
+ {% endif %}
8
+ }{% endmacro %}
9
+ {% macro quant_read_column_zero(fn_name, buffer_name, is_unsigned, present=true, per_column=false, param_name="col", return_type="i32") %}
10
+ fn {{ fn_name }}({% if per_column %}{{ param_name }}: u32{% endif %}) -> {{ return_type }} {
11
+ {% if per_column %}
12
+ {% if is_unsigned %}
13
+ return i32({{ buffer_name }}[{{ param_name }}]);
14
+ {% else %}
15
+ return {{ buffer_name }}[{{ param_name }}];
16
+ {% endif %}
17
+ {% else %}
18
+ {% if is_unsigned %}
19
+ return i32({{ buffer_name }}[0]);
20
+ {% else %}
21
+ return {{ buffer_name }}[0];
22
+ {% endif %}
23
+ {% endif %}
24
+ }{% endmacro %}
25
  {% if not env.device.wgslLanguageFeatures.has("packed_4x8_integer_dot_product") %}
26
  // Runtime source-compatible fallbacks for the packed 4x8 integer built-ins used
27
  // by quantized kernels. Native implementations remain unshadowed when the
 
42
  }
43
  {% endif %}
44
  {{ env.wgsl.resourceDeclarations }}
45
+ {% if requant != "none" %}
46
+ // Rare-boundary fallback: reproduce the reference's binary64 scale division
47
+ // and accumulator product using integer significands; no f64 feature required.
48
+ fn quant_u96_add(a: vec3<u32>, b: vec3<u32>) -> vec3<u32> {
49
+ let low = a.x + b.x;
50
+ let middle = a.y + b.y;
51
+ let with_carry = middle + u32(low < a.x);
52
+ return vec3<u32>(low, with_carry, a.z + b.z + u32(middle < a.y) + u32(with_carry < middle));
53
+ }
54
+ fn quant_u96_shr(a: vec3<u32>) -> vec3<u32> {
55
+ return vec3<u32>((a.x >> 1u) | (a.y << 31u), (a.y >> 1u) | (a.z << 31u), a.z >> 1u);
56
+ }
57
+ fn quant_u96_round_shift(value: vec3<u32>, shift: u32) -> vec3<u32> {
58
+ var q = value;
59
+ var sticky = false;
60
+ var guard = false;
61
+ for (var bit = 0u; bit < shift; bit++) {
62
+ sticky = sticky || guard;
63
+ guard = (q.x & 1u) != 0u;
64
+ q = quant_u96_shr(q);
65
+ }
66
+ if (guard && (sticky || (q.x & 1u) != 0u)) {
67
+ q = quant_u96_add(q, vec3<u32>(1u, 0u, 0u));
68
+ }
69
+ return q;
70
+ }
71
+ fn quant_float_significand(bits: u32) -> u32 {
72
+ return (bits & 0x7fffffu) | select(0x800000u, 0u, (bits & 0x7f800000u) == 0u);
73
+ }
74
+ fn quant_float_exponent(bits: u32) -> i32 {
75
+ return max(1, i32((bits >> 23u) & 255u)) - 150;
76
+ }
77
+ fn quant_round_binary64(acc: i32, a_s: f32, b_s: f32, y_s: f32) -> i32 {
78
+ if (acc == 0) { return 0; }
79
+ let a_bits = bitcast<u32>(a_s);
80
+ let b_bits = bitcast<u32>(b_s);
81
+ let y_bits = bitcast<u32>(y_s);
82
+ let a = quant_float_significand(a_bits);
83
+ let b = quant_float_significand(b_bits);
84
+ let denominator = quant_float_significand(y_bits);
85
+ if (a == 0u || b == 0u) { return 0; }
86
+ // The exact product of two 24-bit significands occupies at most 48 bits.
87
+ let p0 = (a & 65535u) * (b & 65535u);
88
+ let cross = (a >> 16u) * (b & 65535u) + (b >> 16u) * (a & 65535u);
89
+ let product_low = p0 + (cross << 16u);
90
+ let product_high = (a >> 16u) * (b >> 16u) + (cross >> 16u) + u32(product_low < p0);
91
+ var input_bit = select(31 - i32(countLeadingZeros(product_low)), 63 - i32(countLeadingZeros(product_high)), product_high != 0u);
92
+ var remainder = 0u;
93
+ var significant_bits = 0u;
94
+ var leading_exponent = 0;
95
+ var significand = vec2<u32>(0u);
96
+ // Long division generates the 53 bits of the binary64 multiplier. The
97
+ // remainder fits in 25 bits, including its left shift. At most 24 leading
98
+ // zero bits precede the first significant bit.
99
+ loop {
100
+ var incoming = 0u;
101
+ if (input_bit >= 32) { incoming = (product_high >> u32(input_bit - 32)) & 1u; }
102
+ else if (input_bit >= 0) { incoming = (product_low >> u32(input_bit)) & 1u; }
103
+ remainder = (remainder << 1u) | incoming;
104
+ let digit = u32(remainder >= denominator);
105
+ if (digit != 0u) { remainder -= denominator; }
106
+ if (significant_bits != 0u || digit != 0u) {
107
+ if (significant_bits == 0u) { leading_exponent = input_bit; }
108
+ significand = vec2<u32>((significand.x << 1u) | digit, (significand.y << 1u) | (significand.x >> 31u));
109
+ significant_bits++;
110
+ }
111
+ input_bit--;
112
+ if (significant_bits == 53u) { break; }
113
+ }
114
+ let twice_remainder = remainder << 1u;
115
+ if (twice_remainder > denominator || (twice_remainder == denominator && (significand.x & 1u) != 0u)) {
116
+ let low = significand.x + 1u;
117
+ significand = vec2<u32>(low, significand.y + u32(low == 0u));
118
+ }
119
+ let scale_exponent = quant_float_exponent(a_bits) + quant_float_exponent(b_bits) - quant_float_exponent(y_bits) + leading_exponent - 52;
120
+ // abs(INT_MIN) is representable as u32. Multiply the rounded multiplier's
121
+ // significand by the exact accumulator into three little-endian u32 limbs.
122
+ var magnitude = select(u32(acc), 0u - u32(acc), acc < 0);
123
+ var operand = vec3<u32>(significand, 0u);
124
+ var product = vec3<u32>(0u);
125
+ loop {
126
+ if ((magnitude & 1u) != 0u) { product = quant_u96_add(product, operand); }
127
+ magnitude >>= 1u;
128
+ if (magnitude == 0u) { break; }
129
+ operand = vec3<u32>(operand.x << 1u, (operand.y << 1u) | (operand.x >> 31u), (operand.z << 1u) | (operand.y >> 31u));
130
+ }
131
+ var product_top = 31 - i32(countLeadingZeros(product.x));
132
+ if (product.y != 0u) { product_top = 63 - i32(countLeadingZeros(product.y)); }
133
+ if (product.z != 0u) { product_top = 95 - i32(countLeadingZeros(product.z)); }
134
+ if (product_top + scale_exponent >= 10) { return select(1024, -1024, acc < 0); }
135
+ if (product_top + scale_exponent < -2) { return 0; }
136
+ let product_shift = u32(max(0, product_top - 52));
137
+ let rounded_product = quant_u96_round_shift(product, product_shift);
138
+ let result_exponent = scale_exponent + i32(product_shift);
139
+ var rounded = rounded_product;
140
+ if (result_exponent < 0) { rounded = quant_u96_round_shift(rounded_product, u32(-result_exponent)); }
141
+ else { rounded.x <<= u32(result_exponent); }
142
+ return select(i32(rounded.x), -i32(rounded.x), acc < 0);
143
+ }
144
+
145
+ // A power-of-two division and an exactly represented <=48-bit product need
146
+ // no binary64 boundary resolver. This includes ordinary dyadic quantization.
147
+ fn quant_exact_dyadic_product(acc: i32, scale: f32) -> bool {
148
+ let magnitude = select(u32(acc), 0u - u32(acc), acc < 0);
149
+ let significand = quant_float_significand(bitcast<u32>(scale));
150
+ let integer_bits = 32u - countLeadingZeros(magnitude) - countTrailingZeros(magnitude);
151
+ let scale_bits = 32u - countLeadingZeros(significand) - countTrailingZeros(significand);
152
+ return integer_bits + scale_bits <= 48u;
153
+ }
154
+ fn quant_normal_expansion_range(v: f32) -> bool {
155
+ let exponent = (bitcast<u32>(v) >> 23u) & 255u;
156
+ // Reserve two binary32 significands plus four carry/error bits above the
157
+ // minimum normal exponent, so residual arithmetic avoids FTZ underflow.
158
+ return exponent >= (1u + 2u * 24u + 4u) && exponent < 255u;
159
+ }
160
+
161
+ fn round_expansion_ratio(numerator_hi: f32, numerator_lo: f32, denominator: f32, dyadic_scale: f32, dyadic_candidate: bool, acc: i32, a_s: f32, b_s: f32, y_s: f32) -> i32 {
162
+ let estimate = numerator_hi / denominator;
163
+ // Anything outside this range saturates after adding an 8-bit zero point;
164
+ // returning early also keeps floor/i32 conversion away from Inf and overflow.
165
+ if (estimate <= -1024.0) { return -1024; }
166
+ if (estimate >= 1024.0) { return 1024; }
167
+
168
+ let lo = floor(estimate);
169
+ let hi = lo + 1.0;
170
+ let midpoint = lo + 0.5;
171
+ let midpoint_hi = fma(midpoint, denominator, 0.0);
172
+ if ((bitcast<u32>(midpoint_hi) & 0x7f800000u) == 0x7f800000u) { return quant_round_binary64(acc, a_s, b_s, y_s); }
173
+ let midpoint_lo = fma(midpoint, denominator, -midpoint_hi);
174
+ // The high-word subtraction is exact for these similarly sized products
175
+ // (Sterbenz); adding the residual difference therefore preserves its sign
176
+ // even when the rounded high words straddle adjacent f32 values.
177
+ let midpoint_delta = fma(1.0, numerator_hi - midpoint_hi, numerator_lo - midpoint_lo);
178
+ // Escalate ambiguous expansion signs instead of changing the tie tolerance.
179
+ // Six FMA/product tails and their additions fit within this conservative
180
+ // 16*u^2 error envelope (u=2^-24), including binary64 reference rounding.
181
+ if ((bitcast<u32>(midpoint_delta) & 0x7f800000u) == 0x7f800000u) { return quant_round_binary64(acc, a_s, b_s, y_s); }
182
+ if (abs(midpoint_delta) <= max(abs(numerator_hi), abs(midpoint_hi)) * 0x1p-44) {
183
+ if (!dyadic_candidate || !quant_exact_dyadic_product(acc, dyadic_scale)) { return quant_round_binary64(acc, a_s, b_s, y_s); }
184
+ }
185
+ if (midpoint_delta < 0.0) {
186
+ return i32(lo);
187
+ }
188
+ if (midpoint_delta > 0.0) {
189
+ return i32(hi);
190
+ }
191
+ let half_lo = floor(lo * 0.5);
192
+ let lo_is_even = lo - half_lo * 2.0 == 0.0;
193
+ return i32(select(hi, lo, lo_is_even));
194
+ }
195
+
196
+ // Multiplication by a power of two keeps every i32 bit in binary64. Round the
197
+ // exact dyadic value directly, including INT_MIN, half-even ties and saturation.
198
+ fn quant_round_power_two(acc: i32, exponent: i32) -> i32 {
199
+ let magnitude = select(u32(acc), 0u - u32(acc), acc < 0);
200
+ var rounded = 0u;
201
+ if (exponent >= 0) {
202
+ if (exponent >= 10) { return select(1024, -1024, acc < 0); }
203
+ rounded = min(magnitude, 1024u >> u32(exponent)) << u32(exponent);
204
+ } else {
205
+ let shift = u32(-exponent);
206
+ if (shift >= 32u) { return 0; }
207
+ let whole = magnitude >> shift;
208
+ let remainder = magnitude & ((1u << shift) - 1u);
209
+ let half = 1u << (shift - 1u);
210
+ rounded = min(1024u, whole + u32(remainder > half || (remainder == half && (whole & 1u) != 0u)));
211
+ }
212
+ return select(i32(rounded), -i32(rounded), acc < 0);
213
+ }
214
+
215
+ fn round_requantized_precise(acc: i32, a_s: f32, b_s: f32, y_s: f32) -> i32 {
216
+ if (acc == 0) { return 0; }
217
+ let a_bits = bitcast<u32>(a_s);
218
+ let b_bits = bitcast<u32>(b_s);
219
+ let y_bits = bitcast<u32>(y_s);
220
+ if (((a_bits | b_bits | y_bits) & 0x807fffffu) == 0u && min(a_bits, min(b_bits, y_bits)) != 0u && max(a_bits, max(b_bits, y_bits)) < 0x7f800000u) {
221
+ let exponent = i32(a_bits >> 23u) + i32(b_bits >> 23u) - i32(y_bits >> 23u) - 127;
222
+ return quant_round_power_two(acc, exponent);
223
+ }
224
+ let acc_f = f32(acc);
225
+ // Every i32 high part aligned to 256 is exactly representable in f32.
226
+ // Recover the conversion residual without an out-of-range i32(acc_f)
227
+ // conversion when INT_MAX rounds up to 2^31.
228
+ let acc_tail = fma(-1.0, acc_f - f32(acc & -256), f32(acc & 255));
229
+ // Equal b/y scales cancel exactly. Bypass their product and division so a
230
+ // rounded intermediate cannot move a near-half value onto the tie.
231
+ if (bitcast<u32>(b_s) == bitcast<u32>(y_s)) {
232
+ let direct_hi = fma(acc_f, a_s, 0.0);
233
+ let direct_lo = fma(acc_tail, a_s, fma(acc_f, a_s, -direct_hi));
234
+ if (!quant_normal_expansion_range(direct_hi)) { return quant_round_binary64(acc, a_s, b_s, y_s); }
235
+ return round_expansion_ratio(direct_hi, direct_lo, 1.0, a_s, true, acc, a_s, b_s, y_s);
236
+ }
237
+
238
+ // Keep the scale product as a two-float expansion. Its FMA residual preserves
239
+ // which side of a rounding midpoint a near-half value occupies.
240
+ let ab_hi = fma(a_s, b_s, 0.0);
241
+ let ab_lo = fma(a_s, b_s, -ab_hi);
242
+ if (!quant_normal_expansion_range(ab_hi) || !quant_normal_expansion_range(y_s) || y_s > 0x1p126) { return quant_round_binary64(acc, a_s, b_s, y_s); }
243
+ let product_hi = fma(acc_f, ab_hi, 0.0);
244
+ if (!quant_normal_expansion_range(product_hi)) { return quant_round_binary64(acc, a_s, b_s, y_s); }
245
+ let rounded_acc_tail = fma(acc_f, ab_hi, -product_hi) + fma(acc_f, ab_lo, 0.0);
246
+ let product_tail = fma(acc_tail, ab_hi, rounded_acc_tail) + fma(acc_tail, ab_lo, 0.0);
247
+ // Compare the expansion directly. Renormalizing it adds a cancelling
248
+ // subtraction/addition chain that can erase the residual during lowering.
249
+ let denominator_bits = bitcast<u32>(y_s);
250
+ let denominator_power_two = (denominator_bits & 0x7fffffu) == 0u;
251
+ let dyadic_candidate = denominator_power_two && ab_lo == 0.0;
252
+ return round_expansion_ratio(product_hi, product_tail, y_s, ab_hi, dyadic_candidate, acc, a_s, b_s, y_s);
253
+ }
254
 
255
+ // A conservative binary32 error filter accepts only a result safely inside one
256
+ // integer rounding bin. Ambiguous boundaries keep the full expansion/binary64
257
+ // path; this is an exact-result shortcut, not an approximate quantization mode.
258
+ fn round_requantized(acc: i32, a_s: f32, b_s: f32, y_s: f32) -> i32 {
259
+ if (acc == 0) { return 0; }
260
+ let a_bits = bitcast<u32>(a_s);
261
+ let b_bits = bitcast<u32>(b_s);
262
+ let y_bits = bitcast<u32>(y_s);
263
+ let ea = (a_bits >> 23u) & 255u;
264
+ let eb = (b_bits >> 23u) & 255u;
265
+ let ey = (y_bits >> 23u) & 255u;
266
+ // Certify the range before arithmetic: overflow need not produce infinity.
267
+ // Any nonzero i32 converts to a float with biased exponent <=158. These
268
+ // scale-only bounds cover every accumulator and can be shared across lanes.
269
+ // Reserve two significands plus carry bits above FTZ, and a factor of two
270
+ // below overflow for products, division and the expansion midpoint product.
271
+ if (((a_bits | b_bits | y_bits) & 0x80000000u) != 0u ||
272
+ min(ea, eb) == 0u || max(ea, eb) >= 255u ||
273
+ ea + eb < 180u || ea + eb > 347u ||
274
+ ey < 53u || ey > 242u || ea + eb > ey + 220u) {
275
+ return quant_round_binary64(acc, a_s, b_s, y_s);
276
+ }
277
+ let scale = fma(a_s, b_s, 0.0);
278
+ let product = fma(f32(acc), scale, 0.0);
279
+ let estimate = product / y_s;
280
+ if (estimate <= -1024.0) { return -1024; }
281
+ if (estimate >= 1024.0) { return 1024; }
282
+ let nearest = round(estimate);
283
+ // 32 binary32 unit roundoffs reserve room for the i32 conversion, two
284
+ // products, quotient, and distance arithmetic, plus binary64 rounding.
285
+ let margin = max(1.0, abs(estimate)) * 0x1p-19;
286
+ if (abs(estimate - nearest) < 0.5 - margin) { return i32(nearest); }
287
+ {% if device.adapterInfo.vendor != "apple" %}
288
+ // A rounded half-integer estimate can hide which side the exact f32-scale
289
+ // ratio lies on. Resolve that one-ulp neighborhood conservatively outside
290
+ // the compact fast path; exact power-of-two ratios already have a cheap
291
+ // and correct path in round_requantized_precise.
292
+ if (abs(abs(estimate - nearest) - 0.5) <= max(1.0, abs(estimate)) * 0x1p-24) {
293
+ let dyadic_scales = ((a_bits | b_bits | y_bits) & 0x807fffffu) == 0u &&
294
+ min(a_bits, min(b_bits, y_bits)) != 0u &&
295
+ max(a_bits, max(b_bits, y_bits)) < 0x7f800000u;
296
+ if (!dyadic_scales) { return quant_round_binary64(acc, a_s, b_s, y_s); }
297
+ }
298
+ {% endif %}
299
+ return round_requantized_precise(acc, a_s, b_s, y_s);
300
+ }
301
+ {% endif %}
302
 
303
  const TILE_M: u32 = {{ tileM }}u;
304
  const TILE_N: u32 = {{ tileN }}u;
 
341
  return dot4I8Packed(a_word, b_word);
342
  }
343
 
 
344
  fn sum4_b(b0: {{ bScalar }}, b1: {{ bScalar }}, b2: {{ bScalar }}, b3: {{ bScalar }}) -> i32 {
345
  {% if bUnsigned %}
346
  return i32(b0 + b1 + b2 + b3) - 512;
 
349
  {% endif %}
350
  }
351
 
352
+ {{ quant_read_column_zero("read_a_zero", "a_zero_point", aUnsigned, per_column=aZeroPerRow, param_name="row") }}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
353
 
354
+ {{ quant_read_column_zero("read_b_zero", "b_zero_point", bUnsigned, per_column=bZeroPerColumn) }}
 
 
 
 
 
 
355
 
356
  {% if requant != "none" %}
357
+ {{ quant_read_scalar_zero("read_y_zero", "y_zero_point", yUnsigned) }}
 
 
 
 
 
 
358
 
359
  {% endif %}
360
  @compute @workgroup_size(TILE_N, 1, 1)
 
502
  acc = acc + ca * sum_b;
503
  acc = acc + kcc;
504
  {% if hasBias %}
505
+ acc = acc + bias[{{ "col" if biasPerColumn is defined and biasPerColumn else "mr" }}];
506
  {% endif %}
507
  {% if requant == "none" %}
508
  y[y_base + mr * params.N + col] = acc;
509
  {% else %}
510
  {% set scale_index = "col" if requant == "per_column" else ("mr" if requant == "per_row" else "0u") %}
511
+ let rounded = round_requantized(acc, f32(a_scale[0]), f32(b_scale[{{ scale_index }}]), f32(y_scale[0]));
512
+ let q = clamp(rounded + read_y_zero(), {{ qMin }}, {{ qMax }});
 
 
 
513
  {% if yUnsigned %}
514
  y[y_base + mr * params.N + col] = u32(q);
515
  {% else %}
build/webgpu/quant-pad-k-matrices.wgsl.jinja CHANGED
@@ -1,3 +1,11 @@
 
 
 
 
 
 
 
 
1
  // Pad the K dimension of widened quantized matrices for aligned DP4A. A is
2
  // [M, K], optional B is [batch, K, N], and outputs use K_PADDED. Padding stores
3
  // each tensor's raw zero point, so added lanes contribute exactly zero to
@@ -5,13 +13,7 @@
5
  // path instead of a scalar-tail path.
6
  {{ env.wgsl.resourceDeclarations }}
7
 
8
- fn read_a_zero({% if aZeroPerRow %}row: u32{% endif %}) -> i32 {
9
- {% if aZeroPerRow %}
10
- return a_zero_point[row];
11
- {% else %}
12
- return a_zero_point[0];
13
- {% endif %}
14
- }
15
 
16
  @compute @workgroup_size({{ tunables.WORKGROUP_SIZE }})
17
  fn main(
 
1
+ {% macro quant_read_column_zero(fn_name, buffer_name, is_unsigned, present=true, per_column=false, param_name="col", return_type="i32") %}
2
+ fn {{ fn_name }}({% if per_column %}{{ param_name }}: u32{% endif %}) -> {{ return_type }} {
3
+ {% if per_column %}
4
+ return {{ buffer_name }}[{{ param_name }}];
5
+ {% else %}
6
+ return {{ buffer_name }}[0];
7
+ {% endif %}
8
+ }{% endmacro %}
9
  // Pad the K dimension of widened quantized matrices for aligned DP4A. A is
10
  // [M, K], optional B is [batch, K, N], and outputs use K_PADDED. Padding stores
11
  // each tensor's raw zero point, so added lanes contribute exactly zero to
 
13
  // path instead of a scalar-tail path.
14
  {{ env.wgsl.resourceDeclarations }}
15
 
16
+ {{ quant_read_column_zero("read_a_zero", "a_zero_point", false, per_column=aZeroPerRow, param_name="row") }}
 
 
 
 
 
 
17
 
18
  @compute @workgroup_size({{ tunables.WORKGROUP_SIZE }})
19
  fn main(
build/webgpu/test.json CHANGED
@@ -11,7 +11,7 @@
11
  {
12
  "name": "same_upper_stride2_autopad",
13
  "provenance": {
14
- "source": "ONNX Runtime QLinearConv-10 CPUExecutionProvider",
15
  "notes": "QLinearConv inherits Conv auto_pad semantics. This fixture exercises the exact SAME_UPPER spelling and derived asymmetric bottom/right padding; unit scales and zero points make the expected requantized values equal the integer convolution sums."
16
  },
17
  "attrs": { "auto_pad": "SAME_UPPER", "kernel_shape": [3, 3], "strides": [2, 2] },
@@ -228,11 +228,7 @@
228
  "attrs": {}
229
  },
230
  {
231
- "name": "uint8_pointwise_subnormal_scale_ratio_gpu_gap",
232
- "skipGpu": {
233
- "category": "permanent",
234
- "reason": "Portable WGSL floating-point semantics do not guarantee preservation of the subnormal scale values required by this fixture. Backend evidence: Requant multiplier x_scale*w_scale/y_scale = 1e-40/1e-40 divides two denormals; Metal flushes denormals in floating-point division (-> NaN), while the CPU reference computes the ratio with denormal support. Subnormal scale-ratio cases remain CPU-reference-only."
235
- },
236
  "provenance": {
237
  "source": "onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc",
238
  "test": "QLinearConvTest.Conv2DTest",
@@ -252,7 +248,14 @@
252
  "y_scale": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [1e-40] } },
253
  "y_zero_point": { "dtype": "uint8", "shape": [1], "data": { "kind": "values", "values": [0] } }
254
  },
255
- "outputs": { "y": { "dtype": "uint8", "shape": [1, 4, 1, 1], "tolerance": 0 } },
 
 
 
 
 
 
 
256
  "attrs": {}
257
  },
258
  {
@@ -1184,7 +1187,7 @@
1184
  {
1185
  "name": "dp4a_direct_3x3_u8s8_padding_spatial_oc_tails",
1186
  "provenance": {
1187
- "notes": "Compact route lock for the direct tiled 3x3 DP4A kernel. Nonzero zero points exercise exact correction; 10x11 spatial and 12-channel output tails cover partial workgroups."
1188
  },
1189
  "attrs": { "strides": [1, 1], "pads": [1, 1, 1, 1] },
1190
  "inputs": {
@@ -1454,7 +1457,7 @@
1454
  {
1455
  "name": "dp4a_im2col_padded_k81_baseline_exact",
1456
  "provenance": {
1457
- "notes": "Exactness and selector lock for no-feature 2-D im2col with odd K=81; im2col writes raw-x-zero-point padded rows and the shared pad pass extends per-row weight zero points."
1458
  },
1459
  "attrs": { "strides": [1, 1], "pads": [0, 0, 0, 0] },
1460
  "inputs": {
@@ -1483,7 +1486,7 @@
1483
  {
1484
  "name": "dp4a_im2col_bias_tail_k27_baseline_exact",
1485
  "provenance": {
1486
- "notes": "Exact selector lock for the portable DP4A bias path with odd K=27; the shared core packs its final three weight/input lanes without shader-f16."
1487
  },
1488
  "attrs": { "strides": [1, 1], "pads": [0, 0, 0, 0] },
1489
  "inputs": {
@@ -1515,7 +1518,7 @@
1515
  "name": "dp4a_im2col_3d_u8s8_b1c4m8_4x8x8_k3",
1516
  "provenance": {
1517
  "source": "ONNX QLinearConv-10 volumetric convolution",
1518
- "notes": "Locks the 3-D im2col-to-DP4A route, including zero-point-filled spatial padding and per-output-channel quantization."
1519
  },
1520
  "attrs": { "pads": [1, 1, 1, 1, 1, 1] },
1521
  "inputs": {
@@ -1549,7 +1552,7 @@
1549
  {
1550
  "name": "feature_independent_x4_group2_octile4_odd_width_compact",
1551
  "provenance": {
1552
- "notes": "Compact selector and exactness lock for grouped 3x3 QLinearConv with four output channels per group. Odd output width also exercises the feature-independent x4 spatial tail."
1553
  },
1554
  "attrs": { "group": 2, "strides": [1, 1], "pads": [1, 1, 1, 1] },
1555
  "inputs": {
@@ -1583,7 +1586,7 @@
1583
  {
1584
  "name": "same_lower_stride2_autopad",
1585
  "provenance": {
1586
- "source": "ONNX Runtime QLinearConv-10 CPUExecutionProvider",
1587
  "notes": "ONNX auto_pad SAME_LOWER places the odd padding element at the start of the axis. The nonzero leading pad verifies that execution uses the derived padding rather than unrelated explicit pad attributes."
1588
  },
1589
  "attrs": { "auto_pad": "SAME_LOWER", "strides": [2, 2] },
@@ -1690,6 +1693,92 @@
1690
  "tolerance": 0
1691
  }
1692
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1693
  }
1694
  ]
1695
  }
 
11
  {
12
  "name": "same_upper_stride2_autopad",
13
  "provenance": {
14
+ "source": "ONNX Runtime's CPU provider (QLinearConv-10)",
15
  "notes": "QLinearConv inherits Conv auto_pad semantics. This fixture exercises the exact SAME_UPPER spelling and derived asymmetric bottom/right padding; unit scales and zero points make the expected requantized values equal the integer convolution sums."
16
  },
17
  "attrs": { "auto_pad": "SAME_UPPER", "kernel_shape": [3, 3], "strides": [2, 2] },
 
228
  "attrs": {}
229
  },
230
  {
231
+ "name": "uint8_pointwise_subnormal_scale_ratio",
 
 
 
 
232
  "provenance": {
233
  "source": "onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc",
234
  "test": "QLinearConvTest.Conv2DTest",
 
248
  "y_scale": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [1e-40] } },
249
  "y_zero_point": { "dtype": "uint8", "shape": [1], "data": { "kind": "values", "values": [0] } }
250
  },
251
+ "outputs": {
252
+ "y": {
253
+ "dtype": "uint8",
254
+ "shape": [1, 4, 1, 1],
255
+ "tolerance": 0,
256
+ "data": { "kind": "values", "values": [1, 2, 3, 4] }
257
+ }
258
+ },
259
  "attrs": {}
260
  },
261
  {
 
1187
  {
1188
  "name": "dp4a_direct_3x3_u8s8_padding_spatial_oc_tails",
1189
  "provenance": {
1190
+ "notes": "Compact 3x3 quantized convolution with nonzero zero points, 10-by-11 spatial output and 12 output channels checks exact arithmetic at tails."
1191
  },
1192
  "attrs": { "strides": [1, 1], "pads": [1, 1, 1, 1] },
1193
  "inputs": {
 
1457
  {
1458
  "name": "dp4a_im2col_padded_k81_baseline_exact",
1459
  "provenance": {
1460
+ "notes": "Odd K=81 with a nonzero input zero point and a per-channel weight zero point checks exact convolution arithmetic; an 18x18 input with no padding reduces to a 16x16 output over 9 input and 32 output channels."
1461
  },
1462
  "attrs": { "strides": [1, 1], "pads": [0, 0, 0, 0] },
1463
  "inputs": {
 
1486
  {
1487
  "name": "dp4a_im2col_bias_tail_k27_baseline_exact",
1488
  "provenance": {
1489
+ "notes": "Odd K=27 and int32 bias check exact quantized convolution through the final three reduction elements."
1490
  },
1491
  "attrs": { "strides": [1, 1], "pads": [0, 0, 0, 0] },
1492
  "inputs": {
 
1518
  "name": "dp4a_im2col_3d_u8s8_b1c4m8_4x8x8_k3",
1519
  "provenance": {
1520
  "source": "ONNX QLinearConv-10 volumetric convolution",
1521
+ "notes": "Three-dimensional quantized convolution checks zero-point-filled spatial padding and per-output-channel quantization."
1522
  },
1523
  "attrs": { "pads": [1, 1, 1, 1, 1, 1] },
1524
  "inputs": {
 
1552
  {
1553
  "name": "feature_independent_x4_group2_octile4_odd_width_compact",
1554
  "provenance": {
1555
+ "notes": "Compact grouped 3x3 QLinearConv with four output channels per group and an odd output width checks exact quantized results at a spatial tail."
1556
  },
1557
  "attrs": { "group": 2, "strides": [1, 1], "pads": [1, 1, 1, 1] },
1558
  "inputs": {
 
1586
  {
1587
  "name": "same_lower_stride2_autopad",
1588
  "provenance": {
1589
+ "source": "ONNX Runtime's CPU provider (QLinearConv-10)",
1590
  "notes": "ONNX auto_pad SAME_LOWER places the odd padding element at the start of the axis. The nonzero leading pad verifies that execution uses the derived padding rather than unrelated explicit pad attributes."
1591
  },
1592
  "attrs": { "auto_pad": "SAME_LOWER", "strides": [2, 2] },
 
1693
  "tolerance": 0
1694
  }
1695
  }
1696
+ },
1697
+ {
1698
+ "name": "requant_grouped_integer_dot_low_bits",
1699
+ "inputs": {
1700
+ "x": {
1701
+ "dtype": "uint8",
1702
+ "shape": [1, 1040, 1, 4],
1703
+ "data": {
1704
+ "kind": "values",
1705
+ "values": [255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 2, 2, 2, 2, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 2, 2, 2, 2]
1706
+ }
1707
+ },
1708
+ "x_scale": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [1.0] } },
1709
+ "x_zero_point": { "dtype": "uint8", "shape": [1], "data": { "kind": "values", "values": [0] } },
1710
+ "w": {
1711
+ "dtype": "int8",
1712
+ "shape": [8, 520, 1, 1],
1713
+ "data": {
1714
+ "kind": "values",
1715
+ "values": [127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 7, 1, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -7, -1, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 7, 0, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -7, 0, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 7, 1, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -7, -1, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 7, 0, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -7, 0]
1716
+ }
1717
+ },
1718
+ "w_scale": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [1.0] } },
1719
+ "w_zero_point": { "dtype": "int8", "shape": [1], "data": { "kind": "values", "values": [0] } },
1720
+ "y_scale": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [33554432.0] } },
1721
+ "y_zero_point": { "dtype": "uint8", "shape": [1], "data": { "kind": "values", "values": [128] } }
1722
+ },
1723
+ "attrs": { "group": 2 },
1724
+ "outputs": {
1725
+ "y": {
1726
+ "dtype": "uint8",
1727
+ "shape": [1, 8, 1, 4],
1728
+ "tolerance": 0,
1729
+ "relTolerance": 0,
1730
+ "data": {
1731
+ "kind": "values",
1732
+ "values": [129, 129, 129, 129, 127, 127, 127, 127, 128, 128, 128, 128, 128, 128, 128, 128, 129, 129, 129, 129, 127, 127, 127, 127, 128, 128, 128, 128, 128, 128, 128, 128]
1733
+ }
1734
+ }
1735
+ },
1736
+ "provenance": {
1737
+ "notes": "Independent integer stencil sum gives ±(2^24+1) in the full contraction. Explicit nearest-even byte outputs preserve the low-bit distinction; partial/padded pixels are included. No bias or int32 overflow."
1738
+ }
1739
+ },
1740
+ {
1741
+ "name": "requant_direct3x3_integer_dot_low_bits",
1742
+ "inputs": {
1743
+ "x": {
1744
+ "dtype": "uint8",
1745
+ "shape": [1, 64, 3, 3],
1746
+ "data": {
1747
+ "kind": "values",
1748
+ "values": [255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 2, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255]
1749
+ }
1750
+ },
1751
+ "x_scale": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [1.0] } },
1752
+ "x_zero_point": { "dtype": "uint8", "shape": [1], "data": { "kind": "values", "values": [0] } },
1753
+ "w": {
1754
+ "dtype": "int8",
1755
+ "shape": [8, 64, 3, 3],
1756
+ "data": {
1757
+ "kind": "cycle",
1758
+ "values": [127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -7, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
1759
+ }
1760
+ },
1761
+ "w_scale": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [1.0] } },
1762
+ "w_zero_point": { "dtype": "int8", "shape": [1], "data": { "kind": "values", "values": [0] } },
1763
+ "y_scale": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [33554432.0] } },
1764
+ "y_zero_point": { "dtype": "uint8", "shape": [1], "data": { "kind": "values", "values": [128] } }
1765
+ },
1766
+ "attrs": { "pads": [1, 1, 1, 1] },
1767
+ "outputs": {
1768
+ "y": {
1769
+ "dtype": "uint8",
1770
+ "shape": [1, 8, 3, 3],
1771
+ "tolerance": 0,
1772
+ "relTolerance": 0,
1773
+ "data": {
1774
+ "kind": "values",
1775
+ "values": [128, 128, 128, 128, 129, 128, 128, 128, 128, 128, 128, 128, 128, 127, 128, 128, 128, 128, 128, 128, 128, 128, 129, 128, 128, 128, 128, 128, 128, 128, 128, 127, 128, 128, 128, 128, 128, 128, 128, 128, 129, 128, 128, 128, 128, 128, 128, 128, 128, 127, 128, 128, 128, 128, 128, 128, 128, 128, 129, 128, 128, 128, 128, 128, 128, 128, 128, 127, 128, 128, 128, 128]
1776
+ }
1777
+ }
1778
+ },
1779
+ "provenance": {
1780
+ "notes": "Independent integer stencil sum gives ±(2^24+1) in the full contraction. Explicit nearest-even byte outputs preserve the low-bit distinction; partial/padded pixels are included. No bias or int32 overflow."
1781
+ }
1782
  }
1783
  ]
1784
  }