diff --git "a/build/webgpu/manifest.json" "b/build/webgpu/manifest.json" new file mode 100644--- /dev/null +++ "b/build/webgpu/manifest.json" @@ -0,0 +1,5422 @@ +{ + "domain": "com.microsoft", + "name": "QLinearConv", + "sinceVersion": 1, + "inputs": { + "x": { "dtype": "TX" }, + "x_scale": { "dtype": "TF" }, + "x_zero_point": { "dtype": "TX" }, + "w": { "dtype": "TW" }, + "w_scale": { "dtype": "TF" }, + "w_zero_point": { "dtype": "TW" }, + "y_scale": { "dtype": "TF" }, + "y_zero_point": { "dtype": "TY" }, + "B": { "dtype": "int32", "rank": 1, "optional": true } + }, + "outputs": { + "y": { + "dtype": "TY", + "rank": "ranks.x", + "shape": "([dim(shapes.x, 0), expectedOutputWidth, dim(shapes.w, 0)] if ranks.x == 3 else ([dim(shapes.x, 0), expectedOutputHeight, expectedOutputWidth, dim(shapes.w, 0)] if ranks.x == 4 else [dim(shapes.x, 0), expectedOutputDepth, expectedOutputHeight, expectedOutputWidth, dim(shapes.w, 0)])) if channelsLast else ([dim(shapes.x, 0), dim(shapes.w, 0), expectedOutputWidth] if ranks.x == 3 else ([dim(shapes.x, 0), dim(shapes.w, 0), expectedOutputHeight, expectedOutputWidth] if ranks.x == 4 else [dim(shapes.x, 0), dim(shapes.w, 0), expectedOutputDepth, expectedOutputHeight, expectedOutputWidth]))" + } + }, + "attributes": { + "auto_pad": { "default": "NOTSET" }, + "channels_last": { "default": 0 }, + "group": { "default": 1 }, + "dilations": {}, + "kernel_shape": {}, + "pads": {}, + "strides": {} + }, + "attributeConstraints": { "auto_pad": { "values": ["NOTSET", "SAME_UPPER", "SAME_LOWER", "VALID"] } }, + "typeConstraints": { "TX": ["uint8", "int8"], "TW": ["uint8", "int8"], "TY": ["uint8", "int8"], "TF": ["float32"] }, + "tunables": { + "WORKGROUP_SIZE": { "default": 256 }, + "NHWC_OW_TILE": { "default": 4 }, + "NHWC_IM2COL_WORDS": { "default": 2 }, + "EXACT_MATRIX_MIN_INVOCATIONS": { "default": 10240 }, + "NHWC_BIAS_TARGET_WORKGROUPS": { "default": 64 } + }, + "derive": { + "channelsLast": "attrs.channels_last != 0", + "convInChannels": "dim(shapes.x, ranks.x - 1) if channelsLast else dim(shapes.x, 1)", + "convOutChannels": "dim(shapes.w, 0)", + "inputDepth": "(dim(shapes.x, 1) if channelsLast else dim(shapes.x, 2)) if ranks.x == 5 else 1", + "inputHeight": "((dim(shapes.x, 2) if channelsLast else dim(shapes.x, 3)) if ranks.x == 5 else ((dim(shapes.x, 1) if channelsLast else dim(shapes.x, 2)) if ranks.x == 4 else 1))", + "inputWidth": "(dim(shapes.x, ranks.x - 2) if channelsLast else dim(shapes.x, ranks.x - 1)) if ranks.x >= 3 else 1", + "outputDepth": "(dim(shapes.y, 1) if channelsLast else dim(shapes.y, 2)) if ranks.y == 5 else 1", + "outputHeight": "((dim(shapes.y, 2) if channelsLast else dim(shapes.y, 3)) if ranks.y == 5 else ((dim(shapes.y, 1) if channelsLast else dim(shapes.y, 2)) if ranks.y == 4 else 1))", + "outputWidth": "(dim(shapes.y, ranks.y - 2) if channelsLast else dim(shapes.y, ranks.y - 1)) if ranks.y >= 3 else 1", + "kernelDepth": "dim(shapes.w, 2) if ranks.w == 5 else 1", + "kernelHeight": "dim(shapes.w, 3) if ranks.w == 5 else (dim(shapes.w, 2) if ranks.w == 4 else 1)", + "kernelWidth": "dim(shapes.w, ranks.w - 1) if ranks.w >= 3 else 1", + "spatialRank": "ranks.w - 2", + "outSpatial": "outputDepth * outputHeight * outputWidth", + "convK": "dim(shapes.w, 1) * kernelDepth * kernelHeight * kernelWidth", + "dp4aCandidateKWords": 128, + "dp4aAppleCompact": "device.adapterInfo.vendor == \"apple\" and device.features.has(\"subgroups\") and not channelsLast and not present.B and attrs.group == 1 and (kernelHeight > 1 or kernelWidth > 1) and convOutChannels >= outSpatial and convK >= 4 * dp4aCandidateKWords and ceilDiv(convOutChannels, 4) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(outSpatial, 4) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", + "dp4aCompactKWords": "dp4aCandidateKWords if dp4aAppleCompact else 32", + "sgmatM": "dim(shapes.x, 0) * outSpatial if channelsLast else convOutChannels", + "sgmatN": "convOutChannels if channelsLast else outSpatial", + "sgmatTileRows": "sgmatM", + "precastRows": "sgmatM", + "precastContraction": "convK", + "precastCols": "sgmatN", + "precastBatches": "1 if channelsLast else dim(shapes.x, 0)", + "portableWorkgroupCols": 8, + "kernelShapeLengthOk": "not has(attrs, \"kernel_shape\") or (attrs.kernel_shape | length) == spatialRank", + "stridesLengthOk": "not has(attrs, \"strides\") or (attrs.strides | length) == spatialRank", + "dilationsLengthOk": "not has(attrs, \"dilations\") or (attrs.dilations | length) == spatialRank", + "padsLengthOk": "not has(attrs, \"pads\") or (attrs.pads | length) == 2 * spatialRank", + "kernelD": "attrs.kernel_shape[0] if kernelShapeLengthOk and has(attrs, \"kernel_shape\") and spatialRank == 3 else 1", + "kernelH": "attrs.kernel_shape[spatialRank - 2] if kernelShapeLengthOk and has(attrs, \"kernel_shape\") and spatialRank >= 2 else 1", + "kernelW": "attrs.kernel_shape[spatialRank - 1] if kernelShapeLengthOk and has(attrs, \"kernel_shape\") and spatialRank >= 1 else 1", + "strideD": "attrs.strides[0] if stridesLengthOk and has(attrs, \"strides\") and spatialRank == 3 else 1", + "strideH": "attrs.strides[spatialRank - 2] if stridesLengthOk and has(attrs, \"strides\") and spatialRank >= 2 else 1", + "strideW": "attrs.strides[spatialRank - 1] if stridesLengthOk and has(attrs, \"strides\") and spatialRank >= 1 else 1", + "dilationD": "attrs.dilations[0] if dilationsLengthOk and has(attrs, \"dilations\") and spatialRank == 3 else 1", + "dilationH": "attrs.dilations[spatialRank - 2] if dilationsLengthOk and has(attrs, \"dilations\") and spatialRank >= 2 else 1", + "dilationW": "attrs.dilations[spatialRank - 1] if dilationsLengthOk and has(attrs, \"dilations\") and spatialRank >= 1 else 1", + "padFront": "attrs.pads[0] if padsLengthOk and has(attrs, \"pads\") and spatialRank == 3 else 0", + "padTop": "attrs.pads[spatialRank - 2] if padsLengthOk and has(attrs, \"pads\") and spatialRank >= 2 else 0", + "padLeft": "attrs.pads[spatialRank - 1] if padsLengthOk and has(attrs, \"pads\") and spatialRank >= 1 else 0", + "padBack": "attrs.pads[spatialRank] if padsLengthOk and has(attrs, \"pads\") and spatialRank == 3 else 0", + "padBottom": "attrs.pads[2 * spatialRank - 2] if padsLengthOk and has(attrs, \"pads\") and spatialRank >= 2 else 0", + "padRight": "attrs.pads[2 * spatialRank - 1] if padsLengthOk and has(attrs, \"pads\") and spatialRank >= 1 else 0", + "autoPadSame": "attrs.auto_pad == \"SAME_UPPER\" or attrs.auto_pad == \"SAME_LOWER\"", + "autoPadValid": "attrs.auto_pad == \"VALID\"", + "samePadDepth": "max(0, (outputDepth - 1) * strideD + (kernelDepth - 1) * dilationD + 1 - inputDepth)", + "samePadHeight": "max(0, (outputHeight - 1) * strideH + (kernelHeight - 1) * dilationH + 1 - inputHeight)", + "samePadWidth": "max(0, (outputWidth - 1) * strideW + (kernelWidth - 1) * dilationW + 1 - inputWidth)", + "samePadFront": "floor(samePadDepth / 2) if attrs.auto_pad == \"SAME_UPPER\" else samePadDepth - floor(samePadDepth / 2)", + "samePadTop": "floor(samePadHeight / 2) if attrs.auto_pad == \"SAME_UPPER\" else samePadHeight - floor(samePadHeight / 2)", + "samePadLeft": "floor(samePadWidth / 2) if attrs.auto_pad == \"SAME_UPPER\" else samePadWidth - floor(samePadWidth / 2)", + "effectivePadFront": "samePadFront if autoPadSame else (0 if autoPadValid else padFront)", + "effectivePadTop": "samePadTop if autoPadSame else (0 if autoPadValid else padTop)", + "effectivePadLeft": "samePadLeft if autoPadSame else (0 if autoPadValid else padLeft)", + "expectedOutputDepth": "ceil(inputDepth / strideD) if autoPadSame else floor((inputDepth + (0 if autoPadValid else padFront + padBack) - ((kernelDepth - 1) * dilationD + 1)) / strideD) + 1", + "expectedOutputHeight": "ceil(inputHeight / strideH) if autoPadSame else floor((inputHeight + (0 if autoPadValid else padTop + padBottom) - ((kernelHeight - 1) * dilationH + 1)) / strideH) + 1", + "expectedOutputWidth": "ceil(inputWidth / strideW) if autoPadSame else floor((inputWidth + (0 if autoPadValid else padLeft + padRight) - ((kernelWidth - 1) * dilationW + 1)) / strideW) + 1", + "wave32Adapter": "has(device.adapterInfo, \"subgroupMinSize\") and has(device.adapterInfo, \"subgroupMaxSize\") and device.adapterInfo.subgroupMinSize == 32 and device.adapterInfo.subgroupMaxSize == 32", + "canPinSubgroupSize32": "device.features.has(\"subgroups\") and device.features.has(\"subgroup-size-control\") and has(device.adapterInfo, \"subgroupMinSize\") and has(device.adapterInfo, \"subgroupMaxSize\") and device.adapterInfo.subgroupMinSize <= 32 and device.adapterInfo.subgroupMaxSize >= 32", + "pinSubgroupSize32": "canPinSubgroupSize32 and not wave32Adapter", + "wave32Effective": "wave32Adapter or pinSubgroupSize32", + "deviceWorkgroupCap": "min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)", + "sgmatTileN": "64", + "sgmatTileK": "32", + "sgmatColumnGroups": "2", + "sgmatStorageRowGroups": "floor((device.limits.maxComputeWorkgroupStorageSize - sgmatTileN * sgmatTileK * 4) / (16 * sgmatTileK * 4))", + "sgmatRowGroups": "max(1, min(4, floor(deviceWorkgroupCap / (32 * sgmatColumnGroups)), sgmatStorageRowGroups, max(floor(sgmatStorageRowGroups / 2), floor(sgmatTileRows / 16))))", + "sgmatTileM": "16 * sgmatRowGroups", + "sgmatSubgroups": "sgmatRowGroups * sgmatColumnGroups", + "sgmatWorkgroup": "32 * sgmatSubgroups", + "sgmatStagingBytes": "(sgmatTileM + sgmatTileN) * sgmatTileK * 4", + "sgmatScratchBytes": "sgmatSubgroups * 4 * 64 * 4", + "sgmatReuseStaging": "sgmatStagingBytes + sgmatScratchBytes > device.limits.maxComputeWorkgroupStorageSize", + "sgmatSharedBytes": "sgmatStagingBytes + (0 if sgmatReuseStaging else sgmatScratchBytes)", + "sgmatDeviceOk": "sgmatWorkgroup <= deviceWorkgroupCap and sgmatSharedBytes <= device.limits.maxComputeWorkgroupStorageSize", + "precastSubgroups": "max(1, min(4, ceilDiv(precastRows, 16), floor(deviceWorkgroupCap / 32), floor(device.limits.maxComputeWorkgroupStorageSize / (8 * 64 * 4))))", + "precastTileM": "16 * precastSubgroups", + "precastTileN": "4 * 8", + "precastTileK": "4 * 8", + "precastWorkgroup": "32 * precastSubgroups", + "preparedK": "ceilDiv(precastContraction, precastTileK) * precastTileK", + "preparedN": "ceilDiv(precastCols, precastTileN) * precastTileN", + "preparedElements": "ceilDiv(precastRows, precastTileM) * precastTileM * preparedK", + "preparedBElements": "precastBatches * preparedK * preparedN", + "prepareWorkgroup": "min(256, deviceWorkgroupCap)", + "prepareTotalElements": "preparedElements + preparedBElements", + "prepareGroups": "ceilDiv(prepareTotalElements, 4 * prepareWorkgroup)", + "prepareDispatchElements": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535)) * min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535)) * prepareWorkgroup * 4", + "precastResourcesOk": "deviceWorkgroupCap >= precastWorkgroup and precastSubgroups * 8 * 64 * 4 <= device.limits.maxComputeWorkgroupStorageSize", + "precastDispatchOk": "prepareTotalElements <= 4294967295 and prepareDispatchElements <= 4294967296 and ceilDiv(precastCols, precastTileN) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(precastRows, precastTileM) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and prepareGroups <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) * min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", + "precastF32BuffersOk": "preparedElements * 4 <= device.limits.maxStorageBufferBindingSize and preparedElements * 4 <= device.limits.maxBufferSize and preparedBElements * 4 <= device.limits.maxStorageBufferBindingSize and preparedBElements * 4 <= device.limits.maxBufferSize", + "precastF16BuffersOk": "preparedElements * 2 <= device.limits.maxStorageBufferBindingSize and preparedElements * 2 <= device.limits.maxBufferSize and preparedBElements * 2 <= device.limits.maxStorageBufferBindingSize and preparedBElements * 2 <= device.limits.maxBufferSize", + "portableWorkgroupRows": "pow2ceil(min(16, max(4, pow2ceil(ceilDiv(precastRows, 4))), floor(deviceWorkgroupCap / portableWorkgroupCols), device.limits.maxComputeWorkgroupSizeY) + 1) / 2", + "portableTileM": "4 * portableWorkgroupRows", + "portableTileN": "4 * portableWorkgroupCols", + "portableTileK": 32, + "portablePartialK": "floor(16777216 / (255 * 255) / portableTileK) * portableTileK", + "portableGeometryOk": "portableWorkgroupRows >= 4 and portableWorkgroupCols <= device.limits.maxComputeWorkgroupSizeX and portablePartialK >= portableTileK and ceilDiv(precastRows, portableTileM) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", + "spatialAttributeLengthsOk": "kernelShapeLengthOk and stridesLengthOk and dilationsLengthOk and padsLengthOk", + "kernelShapeMatchesWeights": "not has(attrs, \"kernel_shape\") or (kernelW == kernelWidth and (spatialRank < 2 or kernelH == kernelHeight) and (spatialRank < 3 or kernelD == kernelDepth))", + "kernelExtentsOk": "kernelWidth >= 1 and (spatialRank < 2 or kernelHeight >= 1) and (spatialRank < 3 or kernelDepth >= 1)", + "stridesValuesOk": "not has(attrs, \"strides\") or (strideD >= 1 and floor(strideD) == strideD and strideH >= 1 and floor(strideH) == strideH and strideW >= 1 and floor(strideW) == strideW)", + "dilationsValuesOk": "not has(attrs, \"dilations\") or (dilationD >= 1 and floor(dilationD) == dilationD and dilationH >= 1 and floor(dilationH) == dilationH and dilationW >= 1 and floor(dilationW) == dilationW)", + "padsValuesOk": "padFront >= 0 and floor(padFront) == padFront and padTop >= 0 and floor(padTop) == padTop and padLeft >= 0 and floor(padLeft) == padLeft and padBack >= 0 and floor(padBack) == padBack and padBottom >= 0 and floor(padBottom) == padBottom and padRight >= 0 and floor(padRight) == padRight", + "explicitPadsOk": "attrs.auto_pad == \"NOTSET\" or not has(attrs, \"pads\")", + "spatialAttributesOk": "spatialRank >= 1 and spatialRank <= 3 and spatialAttributeLengthsOk and kernelShapeMatchesWeights and kernelExtentsOk and stridesValuesOk and dilationsValuesOk and padsValuesOk and explicitPadsOk", + "xyQuantContract": "(ranks.x_scale == 0 or (ranks.x_scale == 1 and dim(shapes.x_scale, 0) == 1)) and (ranks.y_scale == 0 or (ranks.y_scale == 1 and dim(shapes.y_scale, 0) == 1)) and (ranks.x_zero_point == 0 or (ranks.x_zero_point == 1 and dim(shapes.x_zero_point, 0) == 1)) and (ranks.y_zero_point == 0 or (ranks.y_zero_point == 1 and dim(shapes.y_zero_point, 0) == 1))", + "rank3TensorOk": "spatialAttributesOk and ranks.x == 3 and ranks.w == 3 and ranks.y == 3", + "rank5TensorOk": "spatialAttributesOk and ranks.x == 5 and ranks.w == 5 and ranks.y == 5", + "scalarWeightQuantOk": "(ranks.w_scale == 0 or (ranks.w_scale == 1 and dim(shapes.w_scale, 0) == 1)) and (ranks.w_zero_point == 0 or (ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == 1))", + "yScalar": "\"u32\" if tensorDtypes.y == \"uint8\" else \"i32\"", + "yUnsigned": "tensorDtypes.y == \"uint8\"", + "qMin": "0 if tensorDtypes.y == \"uint8\" else 0 - 128", + "qMax": "255 if tensorDtypes.y == \"uint8\" else 127", + "hasBZero": true, + "packedFeature": "device.wgslLanguageFeatures.has(\"packed_4x8_integer_dot_product\")", + "rank4TensorOk": "spatialAttributesOk and ranks.x == 4 and ranks.w == 4 and ranks.y == 4", + "outputBatchChannelsOk": "dim(shapes.y, 0) == dim(shapes.x, 0) and (dim(shapes.y, ranks.y - 1) if channelsLast else dim(shapes.y, 1)) == convOutChannels", + "groupChannelsOk": "attrs.group >= 1 and dim(shapes.w, 1) * attrs.group == convInChannels and convOutChannels % attrs.group == 0", + "outputSpatialOk": "outputWidth == expectedOutputWidth and (spatialRank < 2 or outputHeight == expectedOutputHeight) and (spatialRank < 3 or outputDepth == expectedOutputDepth)", + "perChannelWeightQuantOk": "ranks.w_scale == 1 and dim(shapes.w_scale, 0) == convOutChannels and ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == convOutChannels", + "flexibleWeightQuantOk": "scalarWeightQuantOk or perChannelWeightQuantOk", + "outImageSize": "1 if channelsLast else outSpatial", + "convKPadded": "ceilDiv(convK, 4) * 4", + "biasContractOk": "present.B and ranks.B == 1 and dim(shapes.B, 0) == convOutChannels", + "gemmDispatchOk": "ceilDiv(outSpatial, 64) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(convOutChannels, 64) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(convOutChannels, 8) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(outSpatial, 8) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and dim(shapes.x, 0) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", + "kMajorWeightStorageOk": "convKPadded * convOutChannels * 4 <= device.limits.maxStorageBufferBindingSize", + "nhwcColsStorageOk": "dim(shapes.x, 0) * outSpatial * convKPadded * 4 <= device.limits.maxStorageBufferBindingSize", + "nchwColsStorageOk": "dim(shapes.x, 0) * convK * outSpatial * 4 <= device.limits.maxStorageBufferBindingSize", + "aUnsigned": "tensorDtypes.x == \"uint8\" if channelsLast else tensorDtypes.w == \"uint8\"", + "bUnsigned": "tensorDtypes.w == \"uint8\" if channelsLast else tensorDtypes.x == \"uint8\"", + "aScalar": "\"u32\" if aUnsigned else \"i32\"", + "bScalar": "\"u32\" if bUnsigned else \"i32\"", + "xScalar": "\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\"", + "wScalar": "\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\"", + "aZeroPerRow": "perChannelWeightQuantOk and not channelsLast", + "bZeroPerColumn": "perChannelWeightQuantOk and channelsLast", + "requant": "(\"per_column\" if channelsLast else \"per_row\") if perChannelWeightQuantOk else \"per_tensor\"", + "wZeroPerChannel": "perChannelWeightQuantOk", + "sgmatDispatchOk": "ceilDiv(sgmatN, sgmatTileN) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(sgmatM, sgmatTileM) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", + "sgmatFullGrid": "floor(sgmatM / sgmatTileM) * floor(sgmatN / sgmatTileN) * (1 if channelsLast else dim(shapes.x, 0))", + "sgmatUsefulInvocations": "sgmatFullGrid * sgmatWorkgroup * convK / (convK + 2 * sgmatTileK)", + "precastFullGrid": "floor(sgmatM / precastTileM) * floor(sgmatN / precastTileN) * (1 if channelsLast else dim(shapes.x, 0))", + "precastUsefulInvocations": "precastFullGrid * precastWorkgroup * convK / (convK + 2 * precastTileK)", + "precastAmortizedInvocations": "precastUsefulInvocations * max(1, (convK - 2 * precastTileK) / precastTileK)", + "precast_pointwise_nchw_ok": "(attrs.channels_last == 0) and (not present.B) and (rank4TensorOk) and (flexibleWeightQuantOk) and (attrs.group == 1) and (kernelHeight == 1) and (kernelWidth == 1) and (strideH == 1) and (strideW == 1) and (padTop == 0 and padBottom == 0) and (padLeft == 0 and padRight == 0) and (dim(shapes.w, 1) == convInChannels) and (outputHeight == inputHeight) and (outputWidth == inputWidth) and (gemmDispatchOk)", + "precast_im2col_nchw_ok": "(attrs.channels_last == 0) and (not present.B) and (rank4TensorOk) and (flexibleWeightQuantOk) and (attrs.group == 1) and (dim(shapes.w, 1) == convInChannels) and (outputSpatialOk) and ((kernelHeight > 1 or kernelWidth > 1)) and (convK >= 64) and (convK <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)) and (nchwColsStorageOk) and (gemmDispatchOk)", + "precast_im2col_nchw_bias_ok": "(attrs.channels_last == 0) and (biasContractOk) and (rank4TensorOk) and (flexibleWeightQuantOk) and (attrs.group == 1) and (dim(shapes.w, 1) == convInChannels) and (outputSpatialOk) and (convK <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)) and (nchwColsStorageOk) and (gemmDispatchOk)", + "precast_pointwise_nhwc_ok": "(attrs.channels_last != 0) and (not present.B) and (rank4TensorOk) and (flexibleWeightQuantOk) and (attrs.group == 1) and (dim(shapes.w, 1) == convInChannels) and (kernelHeight == 1) and (kernelWidth == 1) and (strideH == 1) and (strideW == 1) and (padTop == 0 and padBottom == 0) and (padLeft == 0 and padRight == 0) and (outputHeight == inputHeight) and (outputWidth == inputWidth) and (kMajorWeightStorageOk) and (gemmDispatchOk)", + "precast_pointwise_nhwc_bias_ok": "(attrs.channels_last != 0) and (biasContractOk) and (rank4TensorOk) and (flexibleWeightQuantOk) and (attrs.group == 1) and (dim(shapes.w, 1) == convInChannels) and (kernelHeight == 1) and (kernelWidth == 1) and (strideH == 1) and (strideW == 1) and (padTop == 0 and padBottom == 0) and (padLeft == 0 and padRight == 0) and (outputHeight == inputHeight) and (outputWidth == inputWidth) and (kMajorWeightStorageOk) and (gemmDispatchOk)", + "precast_im2col_nhwc_ok": "(attrs.channels_last != 0) and (not present.B) and (rank4TensorOk) and (flexibleWeightQuantOk) and (attrs.group == 1) and (dim(shapes.w, 1) == convInChannels) and ((kernelHeight > 1 or kernelWidth > 1)) and (outputSpatialOk) and (nhwcColsStorageOk) and (kMajorWeightStorageOk) and (gemmDispatchOk)", + "precast_im2col_nhwc_bias_ok": "(attrs.channels_last != 0) and (biasContractOk) and (rank4TensorOk) and (flexibleWeightQuantOk) and (attrs.group == 1) and (dim(shapes.w, 1) == convInChannels) and ((kernelHeight > 1 or kernelWidth > 1)) and (outputSpatialOk) and (nhwcColsStorageOk) and (kMajorWeightStorageOk) and (gemmDispatchOk)", + "portableLoadsOk": "portableTileM * portableTileK % (4 * portableWorkgroupRows * portableWorkgroupCols) == 0 and portableTileN * portableTileK % (4 * portableWorkgroupRows * portableWorkgroupCols) == 0", + "nhwcOc4UnitColumnThreads": "dim(shapes.y, 0) * outputHeight * outputWidth * ceilDiv(convOutChannels, 4)", + "nhwcOc4ColumnTile": "min(tunables.NHWC_OW_TILE, max(1, floor(nhwcOc4UnitColumnThreads / (tunables.WORKGROUP_SIZE * tunables.NHWC_BIAS_TARGET_WORKGROUPS)))) if present.B else tunables.NHWC_OW_TILE", + "nhwcOc4Threads": "dim(shapes.y, 0) * outputHeight * ceilDiv(outputWidth, nhwcOc4ColumnTile) * ceilDiv(convOutChannels, 4)", + "nhwcOc4Workgroups": "ceilDiv(nhwcOc4Threads, tunables.WORKGROUP_SIZE)", + "nhwcOc4DispatchOk": "min(nhwcOc4Workgroups, DISPATCH_FOLD_WIDTH) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(nhwcOc4Workgroups, DISPATCH_FOLD_WIDTH) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", + "bScalePerColumn": "perChannelWeightQuantOk and channelsLast", + "matrixBatchB": "not channelsLast", + "matrixBiasPerRow": "not channelsLast", + "matrixSubgroupId": true, + "prepareBatchB": "not channelsLast", + "requantize": true, + "wUnsigned": "tensorDtypes.w == \"uint8\"", + "xUnsigned": "tensorDtypes.x == \"uint8\"" + }, + "when": ["outputBatchChannelsOk", "xyQuantContract"], + "bindings": { + "x_w": { "name": "x", "elementType": "$xScalar" }, + "x_zero_point_w": { "name": "x_zero_point", "elementType": "$xScalar", "length": 1 }, + "w_w": { "name": "w", "elementType": "$wScalar" }, + "w_zero_point_w": { "name": "w_zero_point", "elementType": "$wScalar" }, + "x_scale": { "elementType": "f32", "length": 1 }, + "w_scale": { "elementType": "f32" }, + "y_scale": { "elementType": "f32", "length": 1 }, + "y_zero_point_w": { "name": "y_zero_point", "elementType": "$yScalar", "length": 1 }, + "y_w": { "name": "y", "elementType": "$yScalar" }, + "B": { "elementType": "i32" }, + "accum_out": { "scratch": "accum", "name": "y", "elementType": "i32" }, + "accum_in": { "scratch": "accum", "name": "accum", "buffer": "read-only-storage", "elementType": "i32" }, + "accum_params_2d": { + "name": "params", + "struct": [ + { "name": "inChannels", "type": "u32", "value": "convInChannels" }, + { "name": "inH", "type": "u32", "value": "inputHeight" }, + { "name": "inW", "type": "u32", "value": "inputWidth" }, + { "name": "outChannels", "type": "u32", "value": "convOutChannels" }, + { "name": "weightInChannels", "type": "u32", "value": "dim(shapes.w, 1)" }, + { "name": "inChannelsPerGroup", "type": "u32", "value": "convInChannels / attrs.group" }, + { "name": "outChannelsPerGroup", "type": "u32", "value": "convOutChannels / attrs.group" }, + { "name": "kernelH", "type": "u32", "value": "kernelHeight" }, + { "name": "kernelW", "type": "u32", "value": "kernelWidth" }, + { "name": "outH", "type": "u32", "value": "outputHeight" }, + { "name": "outW", "type": "u32", "value": "outputWidth" }, + { "name": "strideH", "type": "u32", "value": "strideH" }, + { "name": "strideW", "type": "u32", "value": "strideW" }, + { "name": "dilationH", "type": "u32", "value": "dilationH" }, + { "name": "dilationW", "type": "u32", "value": "dilationW" }, + { "name": "padH", "type": "i32", "value": "effectivePadTop" }, + { "name": "padW", "type": "i32", "value": "effectivePadLeft" }, + { "name": "count", "type": "u32", "value": "numel(shapes.y)" } + ] + }, + "accum_params_3d": { + "name": "params", + "struct": [ + { "name": "inChannels", "type": "u32", "value": "convInChannels" }, + { "name": "inD", "type": "u32", "value": "inputDepth" }, + { "name": "inH", "type": "u32", "value": "inputHeight" }, + { "name": "inW", "type": "u32", "value": "inputWidth" }, + { "name": "outChannels", "type": "u32", "value": "convOutChannels" }, + { "name": "weightInChannels", "type": "u32", "value": "dim(shapes.w, 1)" }, + { "name": "inChannelsPerGroup", "type": "u32", "value": "convInChannels / attrs.group" }, + { "name": "outChannelsPerGroup", "type": "u32", "value": "convOutChannels / attrs.group" }, + { "name": "kernelD", "type": "u32", "value": "kernelDepth" }, + { "name": "kernelH", "type": "u32", "value": "kernelHeight" }, + { "name": "kernelW", "type": "u32", "value": "kernelWidth" }, + { "name": "outD", "type": "u32", "value": "outputDepth" }, + { "name": "outH", "type": "u32", "value": "outputHeight" }, + { "name": "outW", "type": "u32", "value": "outputWidth" }, + { "name": "strideD", "type": "u32", "value": "strideD" }, + { "name": "strideH", "type": "u32", "value": "strideH" }, + { "name": "strideW", "type": "u32", "value": "strideW" }, + { "name": "dilationD", "type": "u32", "value": "dilationD" }, + { "name": "dilationH", "type": "u32", "value": "dilationH" }, + { "name": "dilationW", "type": "u32", "value": "dilationW" }, + { "name": "padD", "type": "i32", "value": "effectivePadFront" }, + { "name": "padH", "type": "i32", "value": "effectivePadTop" }, + { "name": "padW", "type": "i32", "value": "effectivePadLeft" }, + { "name": "count", "type": "u32", "value": "numel(shapes.y)" } + ] + }, + "requant_params": { + "name": "params", + "struct": [ + { "name": "count", "type": "u32", "value": "numel(shapes.y)" }, + { "name": "outChannels", "type": "u32", "value": "convOutChannels" }, + { "name": "outImageSize", "type": "u32", "value": "outImageSize" } + ] + }, + "requant_params_count": { + "name": "params", + "struct": [{ "name": "count", "type": "u32", "value": "numel(shapes.y)" }] + }, + "x4_params": { + "name": "params", + "struct": [ + { "name": "batch", "type": "u32", "value": "dim(shapes.x, 0)" }, + { "name": "inChannels", "type": "u32", "value": "convInChannels" }, + { "name": "inH", "type": "u32", "value": "inputHeight" }, + { "name": "inW", "type": "u32", "value": "inputWidth" }, + { "name": "outChannels", "type": "u32", "value": "convOutChannels" }, + { "name": "weightInChannels", "type": "u32", "value": "dim(shapes.w, 1)" }, + { "name": "inChannelsPerGroup", "type": "u32", "value": "convInChannels / attrs.group" }, + { "name": "outChannelsPerGroup", "type": "u32", "value": "convOutChannels / attrs.group" }, + { "name": "kernelH", "type": "u32", "value": "kernelHeight" }, + { "name": "kernelW", "type": "u32", "value": "kernelWidth" }, + { "name": "outH", "type": "u32", "value": "outputHeight" }, + { "name": "outW", "type": "u32", "value": "outputWidth" }, + { "name": "strideH", "type": "u32", "value": "strideH" }, + { "name": "strideW", "type": "u32", "value": "strideW" }, + { "name": "dilationH", "type": "u32", "value": "dilationH" }, + { "name": "dilationW", "type": "u32", "value": "dilationW" }, + { "name": "padH", "type": "i32", "value": "effectivePadTop" }, + { "name": "padW", "type": "i32", "value": "effectivePadLeft" } + ] + }, + "oc4_params": { + "name": "params", + "struct": [ + { + "name": "count", + "type": "u32", + "value": "dim(shapes.x, 0) * outputHeight * ceilDiv(outputWidth, nhwcOc4ColumnTile) * ceilDiv(convOutChannels, 4)" + }, + { "name": "ocBlocks", "type": "u32", "value": "ceilDiv(convOutChannels, 4)" }, + { "name": "owGroups", "type": "u32", "value": "ceilDiv(outputWidth, nhwcOc4ColumnTile)" }, + { "name": "inChannels", "type": "u32", "value": "convInChannels" }, + { "name": "inH", "type": "u32", "value": "inputHeight" }, + { "name": "inW", "type": "u32", "value": "inputWidth" }, + { "name": "outChannels", "type": "u32", "value": "convOutChannels" }, + { "name": "weightInChannels", "type": "u32", "value": "dim(shapes.w, 1)" }, + { "name": "inChannelsPerGroup", "type": "u32", "value": "convInChannels / attrs.group" }, + { "name": "outChannelsPerGroup", "type": "u32", "value": "convOutChannels / attrs.group" }, + { "name": "kernelH", "type": "u32", "value": "kernelHeight" }, + { "name": "kernelW", "type": "u32", "value": "kernelWidth" }, + { "name": "outH", "type": "u32", "value": "outputHeight" }, + { "name": "outW", "type": "u32", "value": "outputWidth" }, + { "name": "strideH", "type": "u32", "value": "strideH" }, + { "name": "strideW", "type": "u32", "value": "strideW" }, + { "name": "dilationH", "type": "u32", "value": "dilationH" }, + { "name": "dilationW", "type": "u32", "value": "dilationW" }, + { "name": "padH", "type": "i32", "value": "effectivePadTop" }, + { "name": "padW", "type": "i32", "value": "effectivePadLeft" } + ] + }, + "a_weight": { "arg": "w", "name": "a", "elementType": "$aVec4" }, + "a_zero_point_weight": { "arg": "w_zero_point", "name": "a_zero_point", "elementType": "$aScalar" }, + "b_x": { "arg": "x", "name": "b", "elementType": "$bScalar" }, + "b_zero_point_x": { "arg": "x_zero_point", "name": "b_zero_point", "elementType": "$bScalar", "length": 1 }, + "b_cols_nchw": { "scratch": "cols", "name": "b", "buffer": "read-only-storage", "elementType": "$bScalar" }, + "a_x": { "arg": "x", "name": "a", "elementType": "$aVec4" }, + "a_cols_nhwc": { "scratch": "cols", "name": "a", "buffer": "read-only-storage", "elementType": "$aVec4" }, + "a_zero_point_x": { "arg": "x_zero_point", "name": "a_zero_point", "elementType": "$aScalar", "length": 1 }, + "b_weight_k_major": { + "scratch": "w_k_major", + "name": "b", + "buffer": "read-only-storage", + "elementType": "$bScalar" + }, + "b_zero_point_weight": { "arg": "w_zero_point", "name": "b_zero_point", "elementType": "$bScalar" }, + "a_scale_x": { "arg": "x_scale", "name": "a_scale", "elementType": "f32", "length": 1 }, + "b_scale_w": { "arg": "w_scale", "name": "b_scale", "elementType": "f32" }, + "bias": { "arg": "B", "name": "bias", "elementType": "i32" }, + "w_k_major_out": { "scratch": "w_k_major", "name": "w_k_major", "elementType": "$wScalar" }, + "k_major_params": { + "name": "params", + "struct": [ + { "name": "count", "type": "u32", "value": "convKPadded * convOutChannels" }, + { "name": "outChannels", "type": "u32", "value": "convOutChannels" }, + { "name": "weightInChannels", "type": "u32", "value": "dim(shapes.w, 1)" }, + { "name": "kernelH", "type": "u32", "value": "kernelHeight" }, + { "name": "kernelW", "type": "u32", "value": "kernelWidth" }, + { "name": "kRows", "type": "u32", "value": "convK" } + ] + }, + "k_major_params_exact": { + "name": "params", + "struct": [ + { "name": "count", "type": "u32", "value": "convK * convOutChannels" }, + { "name": "outChannels", "type": "u32", "value": "convOutChannels" }, + { "name": "weightInChannels", "type": "u32", "value": "dim(shapes.w, 1)" }, + { "name": "kernelH", "type": "u32", "value": "kernelHeight" }, + { "name": "kernelW", "type": "u32", "value": "kernelWidth" }, + { "name": "kRows", "type": "u32", "value": "convK" } + ] + }, + "x_im2col": { "name": "x", "elementType": "$xVec4" }, + "cols_out_vec4": { "scratch": "cols", "name": "cols", "elementType": "$colsVec4" }, + "im2col_nhwc_params": { + "name": "params", + "struct": [ + { + "name": "count", + "type": "u32", + "value": "ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)" + }, + { "name": "totalWords", "type": "u32", "value": "dim(shapes.x, 0) * outSpatial * (convKPadded / 4)" }, + { "name": "wordsPerRow", "type": "u32", "value": "convKPadded / 4" }, + { "name": "kRows", "type": "u32", "value": "convK" }, + { "name": "outCount", "type": "u32", "value": "outSpatial" }, + { "name": "inChannels", "type": "u32", "value": "convInChannels" }, + { "name": "inH", "type": "u32", "value": "inputHeight" }, + { "name": "inW", "type": "u32", "value": "inputWidth" }, + { "name": "outW", "type": "u32", "value": "outputWidth" } + ] + }, + "im2col_nchw_params": { + "name": "params", + "struct": [ + { "name": "outCount", "type": "u32", "value": "outSpatial" }, + { "name": "inChannels", "type": "u32", "value": "convInChannels" }, + { "name": "inH", "type": "u32", "value": "inputHeight" }, + { "name": "inW", "type": "u32", "value": "inputWidth" }, + { "name": "kRows", "type": "u32", "value": "convK" }, + { "name": "outW", "type": "u32", "value": "outputWidth" } + ] + }, + "gemm_params_nchw": { + "name": "params", + "struct": [ + { "name": "M", "type": "u32", "value": "convOutChannels" }, + { "name": "N", "type": "u32", "value": "outSpatial" }, + { "name": "K", "type": "u32", "value": "convK" }, + { "name": "aBatchStride4", "type": "u32", "value": 0 }, + { "name": "bBatchStride", "type": "u32", "value": "convK * outSpatial" }, + { "name": "yBatchStride", "type": "u32", "value": "convOutChannels * outSpatial" } + ] + }, + "gemm_params_nhwc_pointwise": { + "name": "params", + "struct": [ + { "name": "M", "type": "u32", "value": "outSpatial" }, + { "name": "N", "type": "u32", "value": "convOutChannels" }, + { "name": "K", "type": "u32", "value": "convK" }, + { + "name": "aBatchStride4", + "type": "u32", + "value": "(outSpatial * convK / 4) if convK % 4 == 0 else (outSpatial * convK)" + }, + { "name": "bBatchStride", "type": "u32", "value": 0 }, + { "name": "yBatchStride", "type": "u32", "value": "convOutChannels * outSpatial" } + ] + }, + "gemm_params_nhwc_im2col": { + "name": "params", + "struct": [ + { "name": "M", "type": "u32", "value": "outSpatial" }, + { "name": "N", "type": "u32", "value": "convOutChannels" }, + { "name": "K", "type": "u32", "value": "convKPadded" }, + { "name": "aBatchStride4", "type": "u32", "value": "outSpatial * convKPadded / 4" }, + { "name": "bBatchStride", "type": "u32", "value": 0 }, + { "name": "yBatchStride", "type": "u32", "value": "convOutChannels * outSpatial" } + ] + }, + "x_b": { "name": "x", "elementType": "$bScalar" }, + "x_zero_point_b": { "name": "x_zero_point", "elementType": "$bScalar", "length": 1 }, + "cols_out_b": { "scratch": "cols", "name": "cols", "elementType": "$bScalar" }, + "gemm_params_nchw_matrix": { + "name": "params", + "struct": [ + { "name": "M", "type": "u32", "value": "sgmatM" }, + { "name": "N", "type": "u32", "value": "outSpatial" }, + { "name": "K", "type": "u32", "value": "convK" }, + { "name": "bBatchStride", "type": "u32", "value": "convK * outSpatial" }, + { "name": "yBatchStride", "type": "u32", "value": "convOutChannels * outSpatial" } + ] + }, + "gemm_params_nhwc_pointwise_matrix": { + "name": "params", + "struct": [ + { "name": "M", "type": "u32", "value": "sgmatM" }, + { "name": "N", "type": "u32", "value": "convOutChannels" }, + { "name": "K", "type": "u32", "value": "convK" } + ] + }, + "gemm_params_nhwc_im2col_matrix": { + "name": "params", + "struct": [ + { "name": "M", "type": "u32", "value": "sgmatM" }, + { "name": "N", "type": "u32", "value": "convOutChannels" }, + { "name": "K", "type": "u32", "value": "convKPadded" } + ] + }, + "precast_aOut": { "scratch": "aFloat", "name": "aFloat", "elementType": "$precastScalar" }, + "precast_bOut": { "scratch": "bFloat", "name": "bFloat", "elementType": "$precastScalar" }, + "precast_a": { "scratch": "aFloat", "name": "a", "buffer": "read-only-storage", "elementType": "$precastScalar" }, + "precast_b": { "scratch": "bFloat", "name": "b", "buffer": "read-only-storage", "elementType": "$precastScalar" }, + "precast_matrix_params": { + "name": "params", + "struct": [ + { "name": "M", "type": "u32", "value": "convOutChannels" }, + { "name": "N", "type": "u32", "value": "outSpatial" }, + { "name": "K", "type": "u32", "value": "convK" }, + { "name": "yBatchStride", "type": "u32", "value": "convOutChannels * outSpatial" } + ] + }, + "precast_prepare_plain": { + "name": "params", + "struct": [ + { "name": "M", "type": "u32", "value": "sgmatM" }, + { "name": "K", "type": "u32", "value": "convK" }, + { "name": "paddedK", "type": "u32", "value": "preparedK" }, + { "name": "count", "type": "u32", "value": "prepareTotalElements" }, + { + "name": "groupsX", + "type": "u32", + "value": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + }, + { "name": "N", "type": "u32", "value": "sgmatN" }, + { "name": "paddedN", "type": "u32", "value": "preparedN" }, + { "name": "aCount", "type": "u32", "value": "preparedElements" } + ] + }, + "precast_prepare_spatial": { + "name": "params", + "struct": [ + { "name": "M", "type": "u32", "value": "sgmatM" }, + { "name": "K", "type": "u32", "value": "convK" }, + { "name": "paddedK", "type": "u32", "value": "preparedK" }, + { "name": "count", "type": "u32", "value": "prepareTotalElements" }, + { + "name": "groupsX", + "type": "u32", + "value": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + }, + { "name": "N", "type": "u32", "value": "sgmatN" }, + { "name": "paddedN", "type": "u32", "value": "preparedN" }, + { "name": "aCount", "type": "u32", "value": "preparedElements" }, + { "name": "inChannels", "type": "u32", "value": "convInChannels" }, + { "name": "inH", "type": "u32", "value": "inputHeight" }, + { "name": "inW", "type": "u32", "value": "inputWidth" }, + { "name": "outW", "type": "u32", "value": "outputWidth" } + ] + }, + "precast_matrix_nhwc": { + "name": "params", + "struct": [ + { "name": "M", "type": "u32", "value": "sgmatM" }, + { "name": "N", "type": "u32", "value": "sgmatN" }, + { "name": "K", "type": "u32", "value": "convK" } + ] + }, + "precast_weight_oihw": { "arg": "w", "name": "b", "elementType": "$bScalar" }, + "precast_prepare_nhwc_plain": { + "name": "params", + "struct": [ + { "name": "M", "type": "u32", "value": "sgmatM" }, + { "name": "K", "type": "u32", "value": "convK" }, + { "name": "paddedK", "type": "u32", "value": "preparedK" }, + { "name": "count", "type": "u32", "value": "prepareTotalElements" }, + { + "name": "groupsX", + "type": "u32", + "value": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + }, + { "name": "N", "type": "u32", "value": "sgmatN" }, + { "name": "paddedN", "type": "u32", "value": "preparedN" }, + { "name": "aCount", "type": "u32", "value": "preparedElements" } + ] + }, + "precast_prepare_nhwc_spatial": { + "name": "params", + "struct": [ + { "name": "M", "type": "u32", "value": "sgmatM" }, + { "name": "K", "type": "u32", "value": "convK" }, + { "name": "paddedK", "type": "u32", "value": "preparedK" }, + { "name": "count", "type": "u32", "value": "prepareTotalElements" }, + { + "name": "groupsX", + "type": "u32", + "value": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + }, + { "name": "N", "type": "u32", "value": "sgmatN" }, + { "name": "paddedN", "type": "u32", "value": "preparedN" }, + { "name": "aCount", "type": "u32", "value": "preparedElements" }, + { "name": "outCount", "type": "u32", "value": "outSpatial" }, + { "name": "inChannels", "type": "u32", "value": "convInChannels" }, + { "name": "inH", "type": "u32", "value": "inputHeight" }, + { "name": "inW", "type": "u32", "value": "inputWidth" }, + { "name": "outW", "type": "u32", "value": "outputWidth" } + ] + }, + "portable_f16_a": { "name": "a", "scratch": "aFloat", "buffer": "read-only-storage", "elementType": "vec4" }, + "portable_f16_b": { "name": "b", "scratch": "bFloat", "buffer": "read-only-storage", "elementType": "vec4" }, + "portable_f32_a": { "name": "a", "scratch": "aFloat", "buffer": "read-only-storage", "elementType": "vec4" }, + "portable_f32_b": { "name": "b", "scratch": "bFloat", "buffer": "read-only-storage", "elementType": "vec4" } + }, + "variants": [ + { + "id": "spatial_nd2_scalar", + "when": ["(rank3TensorOk or rank4TensorOk)", "scalarWeightQuantOk", "not present.B", "groupChannelsOk", "outputSpatialOk"], + "derive": { "hasBias": false, "wScalePerChannel": false, "wZeroPerChannel": false }, + "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }], + "passes": [ + { + "id": "accumulate", + "name": "QLinearConv.Accumulate", + "shader": "conv-int-accumulate-spatial.wgsl.jinja", + "derive": { "spatialRankSpec": 2 }, + "bindings": ["x_w", "x_zero_point_w", "w_w", "w_zero_point_w", "accum_out", "accum_params_2d"], + "dispatch": { + "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + }, + { + "id": "requantize", + "name": "QLinearConv.Requantize", + "shader": "qlinear-conv-requantize.wgsl.jinja", + "bindings": ["accum_in", "x_scale", "w_scale", "y_scale", "y_zero_point_w", "y_w", "requant_params_count"], + "dispatch": { + "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + } + ] + }, + { + "id": "spatial_nd2_scalar_bias", + "when": ["(rank3TensorOk or rank4TensorOk)", "scalarWeightQuantOk", "biasContractOk", "groupChannelsOk", "outputSpatialOk"], + "derive": { "hasBias": true, "wScalePerChannel": false, "wZeroPerChannel": false }, + "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }], + "passes": [ + { + "id": "accumulate", + "name": "QLinearConv.Accumulate", + "shader": "conv-int-accumulate-spatial.wgsl.jinja", + "derive": { "spatialRankSpec": 2 }, + "bindings": ["x_w", "x_zero_point_w", "w_w", "w_zero_point_w", "accum_out", "accum_params_2d"], + "dispatch": { + "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + }, + { + "id": "requantize", + "name": "QLinearConv.Requantize", + "shader": "qlinear-conv-requantize.wgsl.jinja", + "bindings": ["accum_in", "x_scale", "w_scale", "y_scale", "y_zero_point_w", "B", "y_w", "requant_params"], + "dispatch": { + "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + } + ] + }, + { + "id": "spatial_nd2_per_channel", + "when": ["(rank3TensorOk or rank4TensorOk)", "perChannelWeightQuantOk", "not present.B", "groupChannelsOk", "outputSpatialOk"], + "derive": { "hasBias": false, "wScalePerChannel": true, "wZeroPerChannel": true }, + "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }], + "passes": [ + { + "id": "accumulate", + "name": "QLinearConv.Accumulate", + "shader": "conv-int-accumulate-spatial.wgsl.jinja", + "derive": { "spatialRankSpec": 2 }, + "bindings": ["x_w", "x_zero_point_w", "w_w", "w_zero_point_w", "accum_out", "accum_params_2d"], + "dispatch": { + "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + }, + { + "id": "requantize", + "name": "QLinearConv.Requantize", + "shader": "qlinear-conv-requantize.wgsl.jinja", + "bindings": ["accum_in", "x_scale", "w_scale", "y_scale", "y_zero_point_w", "y_w", "requant_params"], + "dispatch": { + "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + } + ] + }, + { + "id": "spatial_nd2_per_channel_bias", + "when": ["(rank3TensorOk or rank4TensorOk)", "perChannelWeightQuantOk", "biasContractOk", "groupChannelsOk", "outputSpatialOk"], + "derive": { "hasBias": true, "wScalePerChannel": true, "wZeroPerChannel": true }, + "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }], + "passes": [ + { + "id": "accumulate", + "name": "QLinearConv.Accumulate", + "shader": "conv-int-accumulate-spatial.wgsl.jinja", + "derive": { "spatialRankSpec": 2 }, + "bindings": ["x_w", "x_zero_point_w", "w_w", "w_zero_point_w", "accum_out", "accum_params_2d"], + "dispatch": { + "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + }, + { + "id": "requantize", + "name": "QLinearConv.Requantize", + "shader": "qlinear-conv-requantize.wgsl.jinja", + "bindings": ["accum_in", "x_scale", "w_scale", "y_scale", "y_zero_point_w", "B", "y_w", "requant_params"], + "dispatch": { + "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + } + ] + }, + { + "id": "spatial_nd3_scalar", + "when": ["rank5TensorOk", "scalarWeightQuantOk", "not present.B", "groupChannelsOk", "outputSpatialOk"], + "derive": { "hasBias": false, "wScalePerChannel": false, "wZeroPerChannel": false }, + "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }], + "passes": [ + { + "id": "accumulate", + "name": "QLinearConv.Accumulate3d", + "shader": "conv-int-accumulate-spatial.wgsl.jinja", + "derive": { "spatialRankSpec": 3 }, + "bindings": ["x_w", "x_zero_point_w", "w_w", "w_zero_point_w", "accum_out", "accum_params_3d"], + "dispatch": { + "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + }, + { + "id": "requantize", + "name": "QLinearConv.Requantize", + "shader": "qlinear-conv-requantize.wgsl.jinja", + "bindings": ["accum_in", "x_scale", "w_scale", "y_scale", "y_zero_point_w", "y_w", "requant_params_count"], + "dispatch": { + "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + } + ] + }, + { + "id": "spatial_nd3_scalar_bias", + "when": ["rank5TensorOk", "scalarWeightQuantOk", "biasContractOk", "groupChannelsOk", "outputSpatialOk"], + "derive": { "hasBias": true, "wScalePerChannel": false, "wZeroPerChannel": false }, + "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }], + "passes": [ + { + "id": "accumulate", + "name": "QLinearConv.Accumulate3d", + "shader": "conv-int-accumulate-spatial.wgsl.jinja", + "derive": { "spatialRankSpec": 3 }, + "bindings": ["x_w", "x_zero_point_w", "w_w", "w_zero_point_w", "accum_out", "accum_params_3d"], + "dispatch": { + "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + }, + { + "id": "requantize", + "name": "QLinearConv.Requantize", + "shader": "qlinear-conv-requantize.wgsl.jinja", + "bindings": ["accum_in", "x_scale", "w_scale", "y_scale", "y_zero_point_w", "B", "y_w", "requant_params"], + "dispatch": { + "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + } + ] + }, + { + "id": "spatial_nd3_per_channel", + "when": ["rank5TensorOk", "perChannelWeightQuantOk", "not present.B", "groupChannelsOk", "outputSpatialOk"], + "derive": { "hasBias": false, "wScalePerChannel": true, "wZeroPerChannel": true }, + "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }], + "passes": [ + { + "id": "accumulate", + "name": "QLinearConv.Accumulate3d", + "shader": "conv-int-accumulate-spatial.wgsl.jinja", + "derive": { "spatialRankSpec": 3 }, + "bindings": ["x_w", "x_zero_point_w", "w_w", "w_zero_point_w", "accum_out", "accum_params_3d"], + "dispatch": { + "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + }, + { + "id": "requantize", + "name": "QLinearConv.Requantize", + "shader": "qlinear-conv-requantize.wgsl.jinja", + "bindings": ["accum_in", "x_scale", "w_scale", "y_scale", "y_zero_point_w", "y_w", "requant_params"], + "dispatch": { + "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + } + ] + }, + { + "id": "spatial_nd3_per_channel_bias", + "when": ["rank5TensorOk", "perChannelWeightQuantOk", "biasContractOk", "groupChannelsOk", "outputSpatialOk"], + "derive": { "hasBias": true, "wScalePerChannel": true, "wZeroPerChannel": true }, + "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }], + "passes": [ + { + "id": "accumulate", + "name": "QLinearConv.Accumulate3d", + "shader": "conv-int-accumulate-spatial.wgsl.jinja", + "derive": { "spatialRankSpec": 3 }, + "bindings": ["x_w", "x_zero_point_w", "w_w", "w_zero_point_w", "accum_out", "accum_params_3d"], + "dispatch": { + "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + }, + { + "id": "requantize", + "name": "QLinearConv.Requantize", + "shader": "qlinear-conv-requantize.wgsl.jinja", + "bindings": ["accum_in", "x_scale", "w_scale", "y_scale", "y_zero_point_w", "B", "y_w", "requant_params"], + "dispatch": { + "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + } + ] + }, + { + "id": "nchw2d_columns_x4", + "priority": 10, + "when": ["attrs.channels_last == 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "groupChannelsOk", "outputSpatialOk", "outputWidth >= 4", "convK >= 4", "numel(shapes.y) >= 32"], + "derive": { + "wScalePerChannel": "perChannelWeightQuantOk", + "ocTile": "4 if convOutChannels >= attrs.group * 4 and convOutChannels % (attrs.group * 4) == 0 else 1" + }, + "passes": [ + { + "id": "main", + "name": "QLinearConv.ColumnsX4", + "shader": "qlinear-conv-nchw-x4.wgsl.jinja", + "bindings": ["x_w", "x_scale", "x_zero_point_w", "w_w", "w_scale", "w_zero_point_w", "y_scale", "y_zero_point_w", "y_w", "x4_params"], + "dispatch": { + "x": "min(ceilDiv((dim(shapes.y, 0) * ceilDiv(convOutChannels, 4 if convOutChannels >= attrs.group * 4 and convOutChannels % (attrs.group * 4) == 0 else 1) * outputHeight * ceilDiv(outputWidth, 4)), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((dim(shapes.y, 0) * ceilDiv(convOutChannels, 4 if convOutChannels >= attrs.group * 4 and convOutChannels % (attrs.group * 4) == 0 else 1) * outputHeight * ceilDiv(outputWidth, 4)), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + } + ] + }, + { + "id": "nhwc2d_channels_x4", + "priority": 10, + "when": ["attrs.channels_last != 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "groupChannelsOk", "outputSpatialOk", "convOutChannels % (attrs.group * 4) == 0", "numel(shapes.y) >= 32"], + "derive": { + "wScalePerChannel": "perChannelWeightQuantOk", + "depthwise": false, + "owTile": "nhwcOc4ColumnTile", + "hasBias": false + }, + "passes": [ + { + "id": "main", + "name": "QLinearConv.ChannelsX4", + "shader": "qlinear-conv-nhwc-oc4.wgsl.jinja", + "bindings": ["x_w", "x_scale", "x_zero_point_w", "w_w", "w_scale", "w_zero_point_w", "y_scale", "y_zero_point_w", "y_w", "oc4_params"], + "dispatch": { + "x": "min(ceilDiv((nhwcOc4Threads), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((nhwcOc4Threads), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + } + ] + }, + { + "id": "nhwc2d_depthwise_x4", + "priority": 10, + "when": ["attrs.channels_last != 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "groupChannelsOk", "outputSpatialOk", "attrs.group == convInChannels", "attrs.group == convOutChannels", "convOutChannels % 4 == 0", "numel(shapes.y) >= 32"], + "derive": { + "wScalePerChannel": "perChannelWeightQuantOk", + "depthwise": true, + "owTile": "nhwcOc4ColumnTile", + "hasBias": false + }, + "passes": [ + { + "id": "main", + "name": "QLinearConv.ChannelsX4", + "shader": "qlinear-conv-nhwc-oc4.wgsl.jinja", + "bindings": ["x_w", "x_scale", "x_zero_point_w", "w_w", "w_scale", "w_zero_point_w", "y_scale", "y_zero_point_w", "y_w", "oc4_params"], + "dispatch": { + "x": "min(ceilDiv((nhwcOc4Threads), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((nhwcOc4Threads), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + } + ] + }, + { + "id": "nhwc2d_channels_x4_bias", + "priority": 10, + "when": ["attrs.channels_last != 0", "biasContractOk", "rank4TensorOk", "flexibleWeightQuantOk", "groupChannelsOk", "outputSpatialOk", "convOutChannels % (attrs.group * 4) == 0", "numel(shapes.y) >= 32", "attrs.group > 1", "device.limits.maxStorageBuffersPerShaderStage >= 10", "tunables.WORKGROUP_SIZE <= deviceWorkgroupCap", "tunables.WORKGROUP_SIZE <= device.limits.maxComputeWorkgroupSizeX", "nhwcOc4DispatchOk"], + "derive": { + "wScalePerChannel": "perChannelWeightQuantOk", + "depthwise": false, + "owTile": "nhwcOc4ColumnTile", + "hasBias": true + }, + "passes": [ + { + "id": "main", + "name": "QLinearConv.ChannelsX4", + "shader": "qlinear-conv-nhwc-oc4.wgsl.jinja", + "bindings": ["x_w", "x_scale", "x_zero_point_w", "w_w", "w_scale", "w_zero_point_w", "y_scale", "y_zero_point_w", "y_w", "oc4_params", "bias"], + "dispatch": { + "x": "min(ceilDiv((nhwcOc4Threads), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((nhwcOc4Threads), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + } + ] + }, + { + "id": "nhwc2d_depthwise_x4_bias", + "priority": 10, + "when": ["attrs.channels_last != 0", "biasContractOk", "rank4TensorOk", "flexibleWeightQuantOk", "groupChannelsOk", "outputSpatialOk", "attrs.group == convInChannels", "attrs.group == convOutChannels", "convOutChannels % 4 == 0", "numel(shapes.y) >= 32", "attrs.group > 1", "device.limits.maxStorageBuffersPerShaderStage >= 10", "tunables.WORKGROUP_SIZE <= deviceWorkgroupCap", "tunables.WORKGROUP_SIZE <= device.limits.maxComputeWorkgroupSizeX", "nhwcOc4DispatchOk"], + "derive": { + "wScalePerChannel": "perChannelWeightQuantOk", + "depthwise": true, + "owTile": "nhwcOc4ColumnTile", + "hasBias": true + }, + "passes": [ + { + "id": "main", + "name": "QLinearConv.ChannelsX4", + "shader": "qlinear-conv-nhwc-oc4.wgsl.jinja", + "bindings": ["x_w", "x_scale", "x_zero_point_w", "w_w", "w_scale", "w_zero_point_w", "y_scale", "y_zero_point_w", "y_w", "oc4_params", "bias"], + "dispatch": { + "x": "min(ceilDiv((nhwcOc4Threads), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((nhwcOc4Threads), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + } + ] + }, + { + "id": "dp4a_pointwise_nchw", + "priority": 20, + "when": ["packedFeature", "attrs.channels_last == 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "kernelHeight == 1", "kernelWidth == 1", "strideH == 1", "strideW == 1", "padTop == 0 and padBottom == 0", "padLeft == 0 and padRight == 0", "dim(shapes.w, 1) == convInChannels", "outputHeight == inputHeight", "outputWidth == inputWidth", "gemmDispatchOk"], + "derive": { + "aPacked": "convK % 4 == 0", + "aVec4": "(\"vec4\" if aUnsigned else \"vec4\") if aPacked else aScalar", + "hasBias": false, + "biasPerColumn": "channelsLast", + "tileM": "4 if dp4aAppleCompact else 8", + "tileN": 64, + "ktWords": "dp4aCompactKWords" + }, + "passes": [ + { + "id": "main", + "name": "QLinearConv.PointwiseDp4a", + "shader": "quant-dp4a-matmul.wgsl.jinja", + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "gemm_params_nchw"], + "dispatch": { + "x": "ceilDiv(outSpatial, 64)", + "y": "ceilDiv(convOutChannels, 4 if dp4aAppleCompact else 8)", + "z": "dim(shapes.y, 0)" + } + } + ] + }, + { + "id": "matrix_pointwise_nchw", + "priority": 30, + "when": ["attrs.channels_last == 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "kernelHeight == 1", "kernelWidth == 1", "strideH == 1", "strideW == 1", "padTop == 0 and padBottom == 0", "padLeft == 0 and padRight == 0", "dim(shapes.w, 1) == convInChannels", "outputHeight == inputHeight", "outputWidth == inputWidth", "gemmDispatchOk", "wave32Effective", "sgmatDeviceOk", "sgmatDispatchOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["not (sgmatM >= sgmatTileM and sgmatN >= sgmatTileN / 2 and convK >= sgmatTileK)", "sgmatUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix"], + "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 9 } + }, + "passes": [ + { + "id": "main", + "name": "QLinearConv.ExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "derive": { + "aVec4": "aScalar", + "matrixHasBias": false, + "matrixScalePerRow": "perChannelWeightQuantOk and not channelsLast", + "matrixZeroPerRow": "perChannelWeightQuantOk and not channelsLast" + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "gemm_params_nchw_matrix"], + "dispatch": { + "x": "ceilDiv(sgmatN, sgmatTileN)", + "y": "ceilDiv(sgmatM, sgmatTileM)", + "z": "dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "dp4a_im2col_nchw", + "priority": 20, + "when": ["packedFeature", "attrs.channels_last == 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "outputSpatialOk", "(kernelHeight > 1 or kernelWidth > 1)", "convK >= 64", "convK <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "nchwColsStorageOk", "gemmDispatchOk", "numel(shapes.y) >= 8192"], + "derive": { + "aPacked": "convK % 4 == 0", + "aVec4": "(\"vec4\" if aUnsigned else \"vec4\") if aPacked else aScalar", + "hasBias": false, + "biasPerColumn": "channelsLast", + "tileM": "4 if dp4aAppleCompact else 8", + "tileN": 64, + "ktWords": "dp4aCompactKWords" + }, + "intermediates": [{ "id": "cols", "dtype": "int32", "shape": "[dim(shapes.x, 0) * convK * outSpatial]" }], + "passes": [ + { + "id": "im2col", + "name": "QLinearConv.Im2Col", + "shader": "conv-int-im2col-spatial.wgsl.jinja", + "derive": { + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft" + }, + "bindings": ["x_b", "x_zero_point_b", "cols_out_b", "im2col_nchw_params"], + "dispatch": { "x": "ceilDiv(outSpatial, tunables.WORKGROUP_SIZE)", "y": "convK", "z": "dim(shapes.x, 0)" } + }, + { + "id": "main", + "name": "QLinearConv.Im2ColDp4a", + "shader": "quant-dp4a-matmul.wgsl.jinja", + "bindings": ["a_weight", "b_cols_nchw", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "gemm_params_nchw"], + "dispatch": { + "x": "ceilDiv(outSpatial, 64)", + "y": "ceilDiv(convOutChannels, 4 if dp4aAppleCompact else 8)", + "z": "dim(shapes.y, 0)" + } + } + ] + }, + { + "id": "matrix_im2col_nchw", + "priority": 30, + "when": ["attrs.channels_last == 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "outputSpatialOk", "(kernelHeight > 1 or kernelWidth > 1)", "convK >= 64", "convK <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "nchwColsStorageOk", "gemmDispatchOk", "wave32Effective", "sgmatDeviceOk", "sgmatDispatchOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["not (sgmatM >= sgmatTileM and sgmatN >= sgmatTileN / 2 and convK >= sgmatTileK)", "sgmatUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix"], + "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 9 } + }, + "intermediates": [{ "id": "cols", "dtype": "int32", "shape": "[dim(shapes.x, 0) * convK * outSpatial]" }], + "passes": [ + { + "id": "im2col", + "name": "QLinearConv.Im2Col", + "shader": "conv-int-im2col-spatial.wgsl.jinja", + "derive": { + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft" + }, + "bindings": ["x_b", "x_zero_point_b", "cols_out_b", "im2col_nchw_params"], + "dispatch": { "x": "ceilDiv(outSpatial, tunables.WORKGROUP_SIZE)", "y": "convK", "z": "dim(shapes.x, 0)" } + }, + { + "id": "main", + "name": "QLinearConv.ExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "derive": { + "aVec4": "aScalar", + "matrixHasBias": false, + "matrixScalePerRow": "perChannelWeightQuantOk and not channelsLast", + "matrixZeroPerRow": "perChannelWeightQuantOk and not channelsLast" + }, + "bindings": ["a_weight", "b_cols_nchw", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "gemm_params_nchw_matrix"], + "dispatch": { + "x": "ceilDiv(sgmatN, sgmatTileN)", + "y": "ceilDiv(sgmatM, sgmatTileM)", + "z": "dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "dp4a_im2col_nchw_bias", + "priority": 25, + "when": ["packedFeature", "attrs.channels_last == 0", "biasContractOk", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "outputSpatialOk", "convK <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "nchwColsStorageOk", "gemmDispatchOk"], + "derive": { + "aPacked": "convK % 4 == 0", + "aVec4": "(\"vec4\" if aUnsigned else \"vec4\") if aPacked else aScalar", + "hasBias": true, + "biasPerColumn": "channelsLast", + "tileM": "4 if dp4aAppleCompact else 8", + "tileN": 64, + "ktWords": "dp4aCompactKWords" + }, + "intermediates": [{ "id": "cols", "dtype": "int32", "shape": "[dim(shapes.x, 0) * convK * outSpatial]" }], + "passes": [ + { + "id": "im2col", + "name": "QLinearConv.Im2Col", + "shader": "conv-int-im2col-spatial.wgsl.jinja", + "derive": { + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft" + }, + "bindings": ["x_b", "x_zero_point_b", "cols_out_b", "im2col_nchw_params"], + "dispatch": { "x": "ceilDiv(outSpatial, tunables.WORKGROUP_SIZE)", "y": "convK", "z": "dim(shapes.x, 0)" } + }, + { + "id": "main", + "name": "QLinearConv.Im2ColDp4aBias", + "shader": "quant-dp4a-matmul.wgsl.jinja", + "bindings": ["a_weight", "b_cols_nchw", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "gemm_params_nchw"], + "dispatch": { + "x": "ceilDiv(outSpatial, 64)", + "y": "ceilDiv(convOutChannels, 4 if dp4aAppleCompact else 8)", + "z": "dim(shapes.y, 0)" + } + } + ] + }, + { + "id": "matrix_im2col_nchw_bias", + "priority": 35, + "when": ["attrs.channels_last == 0", "biasContractOk", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "outputSpatialOk", "convK <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "nchwColsStorageOk", "gemmDispatchOk", "wave32Effective", "sgmatDeviceOk", "sgmatDispatchOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["not (sgmatM >= sgmatTileM and sgmatN >= sgmatTileN / 2 and convK >= sgmatTileK)", "sgmatUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix"], + "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 10 } + }, + "intermediates": [{ "id": "cols", "dtype": "int32", "shape": "[dim(shapes.x, 0) * convK * outSpatial]" }], + "passes": [ + { + "id": "im2col", + "name": "QLinearConv.Im2Col", + "shader": "conv-int-im2col-spatial.wgsl.jinja", + "derive": { + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft" + }, + "bindings": ["x_b", "x_zero_point_b", "cols_out_b", "im2col_nchw_params"], + "dispatch": { "x": "ceilDiv(outSpatial, tunables.WORKGROUP_SIZE)", "y": "convK", "z": "dim(shapes.x, 0)" } + }, + { + "id": "main", + "name": "QLinearConv.ExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "derive": { + "aVec4": "aScalar", + "matrixHasBias": true, + "matrixScalePerRow": "perChannelWeightQuantOk and not channelsLast", + "matrixZeroPerRow": "perChannelWeightQuantOk and not channelsLast" + }, + "bindings": ["a_weight", "b_cols_nchw", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "gemm_params_nchw_matrix"], + "dispatch": { + "x": "ceilDiv(sgmatN, sgmatTileN)", + "y": "ceilDiv(sgmatM, sgmatTileM)", + "z": "dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "dp4a_pointwise_nhwc", + "priority": 20, + "when": ["packedFeature", "attrs.channels_last != 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "kernelHeight == 1", "kernelWidth == 1", "strideH == 1", "strideW == 1", "padTop == 0 and padBottom == 0", "padLeft == 0 and padRight == 0", "outputHeight == inputHeight", "outputWidth == inputWidth", "kMajorWeightStorageOk", "gemmDispatchOk", "numel(shapes.y) >= 8192"], + "derive": { + "aPacked": "convK % 4 == 0", + "aVec4": "(\"vec4\" if aUnsigned else \"vec4\") if aPacked else aScalar", + "hasBias": false, + "biasPerColumn": "channelsLast", + "tileM": "4 if dp4aAppleCompact else 8", + "tileN": 64, + "ktWords": "dp4aCompactKWords" + }, + "intermediates": [{ "id": "w_k_major", "dtype": "int32", "shape": "[convK * convOutChannels]" }], + "passes": [ + { + "id": "weights", + "name": "QLinearConv.WeightKMajor", + "shader": "quant-weight-k-major.wgsl.jinja", + "bindings": ["w_w", "w_zero_point_w", "w_k_major_out", "k_major_params_exact"], + "dispatch": { + "x": "min(ceilDiv((convK * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((convK * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + }, + { + "id": "main", + "name": "QLinearConv.NhwcDp4a", + "shader": "quant-dp4a-matmul.wgsl.jinja", + "bindings": ["a_x", "b_weight_k_major", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "gemm_params_nhwc_pointwise"], + "dispatch": { + "x": "ceilDiv(convOutChannels, 64)", + "y": "ceilDiv(outSpatial, 4 if dp4aAppleCompact else 8)", + "z": "dim(shapes.y, 0)" + } + } + ] + }, + { + "id": "matrix_pointwise_nhwc", + "priority": 30, + "when": ["attrs.channels_last != 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "kernelHeight == 1", "kernelWidth == 1", "strideH == 1", "strideW == 1", "padTop == 0 and padBottom == 0", "padLeft == 0 and padRight == 0", "outputHeight == inputHeight", "outputWidth == inputWidth", "kMajorWeightStorageOk", "gemmDispatchOk", "wave32Effective", "sgmatDeviceOk", "sgmatDispatchOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["not (sgmatM >= sgmatTileM and sgmatN >= sgmatTileN / 2 and convK >= sgmatTileK)", "sgmatUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix"], + "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 9 } + }, + "intermediates": [{ "id": "w_k_major", "dtype": "int32", "shape": "[convK * convOutChannels]" }], + "passes": [ + { + "id": "weights", + "name": "QLinearConv.WeightKMajor", + "shader": "quant-weight-k-major.wgsl.jinja", + "bindings": ["w_w", "w_zero_point_w", "w_k_major_out", "k_major_params_exact"], + "dispatch": { + "x": "min(ceilDiv((convK * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((convK * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + }, + { + "id": "main", + "name": "QLinearConv.ExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "derive": { + "aVec4": "aScalar", + "matrixHasBias": false, + "matrixScalePerRow": "perChannelWeightQuantOk and not channelsLast", + "matrixZeroPerRow": "perChannelWeightQuantOk and not channelsLast" + }, + "bindings": ["a_x", "b_weight_k_major", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "gemm_params_nhwc_pointwise_matrix"], + "dispatch": { "x": "ceilDiv(sgmatN, sgmatTileN)", "y": "ceilDiv(sgmatM, sgmatTileM)" } + } + ] + }, + { + "id": "dp4a_pointwise_nhwc_bias", + "priority": 25, + "when": ["packedFeature", "attrs.channels_last != 0", "biasContractOk", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "kernelHeight == 1", "kernelWidth == 1", "strideH == 1", "strideW == 1", "padTop == 0 and padBottom == 0", "padLeft == 0 and padRight == 0", "outputHeight == inputHeight", "outputWidth == inputWidth", "kMajorWeightStorageOk", "gemmDispatchOk"], + "derive": { + "aPacked": "convK % 4 == 0", + "aVec4": "(\"vec4\" if aUnsigned else \"vec4\") if aPacked else aScalar", + "hasBias": true, + "biasPerColumn": "channelsLast", + "tileM": "4 if dp4aAppleCompact else 8", + "tileN": 64, + "ktWords": "dp4aCompactKWords" + }, + "intermediates": [{ "id": "w_k_major", "dtype": "int32", "shape": "[convK * convOutChannels]" }], + "passes": [ + { + "id": "weights", + "name": "QLinearConv.WeightKMajor", + "shader": "quant-weight-k-major.wgsl.jinja", + "bindings": ["w_w", "w_zero_point_w", "w_k_major_out", "k_major_params_exact"], + "dispatch": { + "x": "min(ceilDiv((convK * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((convK * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + }, + { + "id": "main", + "name": "QLinearConv.NhwcDp4aBias", + "shader": "quant-dp4a-matmul.wgsl.jinja", + "bindings": ["a_x", "b_weight_k_major", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "gemm_params_nhwc_pointwise"], + "dispatch": { + "x": "ceilDiv(convOutChannels, 64)", + "y": "ceilDiv(outSpatial, 4 if dp4aAppleCompact else 8)", + "z": "dim(shapes.y, 0)" + } + } + ] + }, + { + "id": "matrix_pointwise_nhwc_bias", + "priority": 35, + "when": ["attrs.channels_last != 0", "biasContractOk", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "kernelHeight == 1", "kernelWidth == 1", "strideH == 1", "strideW == 1", "padTop == 0 and padBottom == 0", "padLeft == 0 and padRight == 0", "outputHeight == inputHeight", "outputWidth == inputWidth", "kMajorWeightStorageOk", "gemmDispatchOk", "wave32Effective", "sgmatDeviceOk", "sgmatDispatchOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["not (sgmatM >= sgmatTileM and sgmatN >= sgmatTileN / 2 and convK >= sgmatTileK)", "sgmatUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix"], + "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 10 } + }, + "intermediates": [{ "id": "w_k_major", "dtype": "int32", "shape": "[convK * convOutChannels]" }], + "passes": [ + { + "id": "weights", + "name": "QLinearConv.WeightKMajor", + "shader": "quant-weight-k-major.wgsl.jinja", + "bindings": ["w_w", "w_zero_point_w", "w_k_major_out", "k_major_params_exact"], + "dispatch": { + "x": "min(ceilDiv((convK * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((convK * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + }, + { + "id": "main", + "name": "QLinearConv.ExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "derive": { + "aVec4": "aScalar", + "matrixHasBias": true, + "matrixScalePerRow": "perChannelWeightQuantOk and not channelsLast", + "matrixZeroPerRow": "perChannelWeightQuantOk and not channelsLast" + }, + "bindings": ["a_x", "b_weight_k_major", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "gemm_params_nhwc_pointwise_matrix"], + "dispatch": { "x": "ceilDiv(sgmatN, sgmatTileN)", "y": "ceilDiv(sgmatM, sgmatTileM)" } + } + ] + }, + { + "id": "dp4a_im2col_nhwc", + "priority": 20, + "when": ["packedFeature", "attrs.channels_last != 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "(kernelHeight > 1 or kernelWidth > 1)", "outputSpatialOk", "nhwcColsStorageOk", "kMajorWeightStorageOk", "gemmDispatchOk", "numel(shapes.y) >= 8192"], + "derive": { + "aPacked": true, + "aVec4": "(\"vec4\" if aUnsigned else \"vec4\") if aPacked else aScalar", + "hasBias": false, + "biasPerColumn": "channelsLast", + "tileM": "4 if dp4aAppleCompact else 8", + "tileN": 64, + "ktWords": "dp4aCompactKWords" + }, + "intermediates": [ + { "id": "w_k_major", "dtype": "int32", "shape": "[convKPadded * convOutChannels]" }, + { "id": "cols", "dtype": "int32", "shape": "[dim(shapes.x, 0) * outSpatial * convKPadded]" } + ], + "passes": [ + { + "id": "weights", + "name": "QLinearConv.WeightKMajor", + "shader": "quant-weight-k-major.wgsl.jinja", + "bindings": ["w_w", "w_zero_point_w", "w_k_major_out", "k_major_params"], + "dispatch": { + "x": "min(ceilDiv((convKPadded * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((convKPadded * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + }, + { + "id": "im2col", + "name": "QLinearConv.Im2ColNhwc", + "shader": "qlinear-conv-im2col-nhwc.wgsl.jinja", + "derive": { + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "im2colWords": "tunables.NHWC_IM2COL_WORDS", + "channelsAligned": "convInChannels % 4 == 0", + "xVec4": "(\"vec4\" if tensorDtypes.x == \"uint8\" else \"vec4\") if convInChannels % 4 == 0 else (\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\")", + "colsVec4": "\"vec4\" if tensorDtypes.x == \"uint8\" else \"vec4\"" + }, + "bindings": ["x_im2col", "x_zero_point_w", "cols_out_vec4", "im2col_nhwc_params"], + "dispatch": { + "x": "min(ceilDiv((ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + }, + { + "id": "main", + "name": "QLinearConv.NhwcDp4a", + "shader": "quant-dp4a-matmul.wgsl.jinja", + "bindings": ["a_cols_nhwc", "b_weight_k_major", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "gemm_params_nhwc_im2col"], + "dispatch": { + "x": "ceilDiv(convOutChannels, 64)", + "y": "ceilDiv(outSpatial, 4 if dp4aAppleCompact else 8)", + "z": "dim(shapes.y, 0)" + } + } + ] + }, + { + "id": "matrix_im2col_nhwc", + "priority": 30, + "when": ["attrs.channels_last != 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "(kernelHeight > 1 or kernelWidth > 1)", "outputSpatialOk", "nhwcColsStorageOk", "kMajorWeightStorageOk", "gemmDispatchOk", "wave32Effective", "sgmatDeviceOk", "sgmatDispatchOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["not (sgmatM >= sgmatTileM and sgmatN >= sgmatTileN / 2 and convK >= sgmatTileK)", "sgmatUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix"], + "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 9 } + }, + "intermediates": [ + { "id": "w_k_major", "dtype": "int32", "shape": "[convKPadded * convOutChannels]" }, + { "id": "cols", "dtype": "int32", "shape": "[dim(shapes.x, 0) * outSpatial * convKPadded]" } + ], + "passes": [ + { + "id": "weights", + "name": "QLinearConv.WeightKMajor", + "shader": "quant-weight-k-major.wgsl.jinja", + "bindings": ["w_w", "w_zero_point_w", "w_k_major_out", "k_major_params"], + "dispatch": { + "x": "min(ceilDiv((convKPadded * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((convKPadded * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + }, + { + "id": "im2col", + "name": "QLinearConv.Im2ColNhwc", + "shader": "qlinear-conv-im2col-nhwc.wgsl.jinja", + "derive": { + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "im2colWords": "tunables.NHWC_IM2COL_WORDS", + "channelsAligned": "convInChannels % 4 == 0", + "xVec4": "(\"vec4\" if tensorDtypes.x == \"uint8\" else \"vec4\") if convInChannels % 4 == 0 else (\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\")", + "colsVec4": "\"vec4\" if tensorDtypes.x == \"uint8\" else \"vec4\"" + }, + "bindings": ["x_im2col", "x_zero_point_w", "cols_out_vec4", "im2col_nhwc_params"], + "dispatch": { + "x": "min(ceilDiv((ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + }, + { + "id": "main", + "name": "QLinearConv.ExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "derive": { + "aVec4": "aScalar", + "matrixHasBias": false, + "matrixScalePerRow": "perChannelWeightQuantOk and not channelsLast", + "matrixZeroPerRow": "perChannelWeightQuantOk and not channelsLast" + }, + "bindings": ["a_cols_nhwc", "b_weight_k_major", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "gemm_params_nhwc_im2col_matrix"], + "dispatch": { "x": "ceilDiv(sgmatN, sgmatTileN)", "y": "ceilDiv(sgmatM, sgmatTileM)" } + } + ] + }, + { + "id": "dp4a_im2col_nhwc_bias", + "priority": 25, + "when": ["packedFeature", "attrs.channels_last != 0", "biasContractOk", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "(kernelHeight > 1 or kernelWidth > 1)", "outputSpatialOk", "nhwcColsStorageOk", "kMajorWeightStorageOk", "gemmDispatchOk"], + "derive": { + "aPacked": true, + "aVec4": "(\"vec4\" if aUnsigned else \"vec4\") if aPacked else aScalar", + "hasBias": true, + "biasPerColumn": "channelsLast", + "tileM": "4 if dp4aAppleCompact else 8", + "tileN": 64, + "ktWords": "dp4aCompactKWords" + }, + "intermediates": [ + { "id": "w_k_major", "dtype": "int32", "shape": "[convKPadded * convOutChannels]" }, + { "id": "cols", "dtype": "int32", "shape": "[dim(shapes.x, 0) * outSpatial * convKPadded]" } + ], + "passes": [ + { + "id": "weights", + "name": "QLinearConv.WeightKMajor", + "shader": "quant-weight-k-major.wgsl.jinja", + "bindings": ["w_w", "w_zero_point_w", "w_k_major_out", "k_major_params"], + "dispatch": { + "x": "min(ceilDiv((convKPadded * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((convKPadded * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + }, + { + "id": "im2col", + "name": "QLinearConv.Im2ColNhwc", + "shader": "qlinear-conv-im2col-nhwc.wgsl.jinja", + "derive": { + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "im2colWords": "tunables.NHWC_IM2COL_WORDS", + "channelsAligned": "convInChannels % 4 == 0", + "xVec4": "(\"vec4\" if tensorDtypes.x == \"uint8\" else \"vec4\") if convInChannels % 4 == 0 else (\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\")", + "colsVec4": "\"vec4\" if tensorDtypes.x == \"uint8\" else \"vec4\"" + }, + "bindings": ["x_im2col", "x_zero_point_w", "cols_out_vec4", "im2col_nhwc_params"], + "dispatch": { + "x": "min(ceilDiv((ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + }, + { + "id": "main", + "name": "QLinearConv.NhwcDp4aBias", + "shader": "quant-dp4a-matmul.wgsl.jinja", + "bindings": ["a_cols_nhwc", "b_weight_k_major", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "gemm_params_nhwc_im2col"], + "dispatch": { + "x": "ceilDiv(convOutChannels, 64)", + "y": "ceilDiv(outSpatial, 4 if dp4aAppleCompact else 8)", + "z": "dim(shapes.y, 0)" + } + } + ] + }, + { + "id": "matrix_im2col_nhwc_bias", + "priority": 35, + "when": ["attrs.channels_last != 0", "biasContractOk", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "(kernelHeight > 1 or kernelWidth > 1)", "outputSpatialOk", "nhwcColsStorageOk", "kMajorWeightStorageOk", "gemmDispatchOk", "wave32Effective", "sgmatDeviceOk", "sgmatDispatchOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["not (sgmatM >= sgmatTileM and sgmatN >= sgmatTileN / 2 and convK >= sgmatTileK)", "sgmatUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix"], + "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 10 } + }, + "intermediates": [ + { "id": "w_k_major", "dtype": "int32", "shape": "[convKPadded * convOutChannels]" }, + { "id": "cols", "dtype": "int32", "shape": "[dim(shapes.x, 0) * outSpatial * convKPadded]" } + ], + "passes": [ + { + "id": "weights", + "name": "QLinearConv.WeightKMajor", + "shader": "quant-weight-k-major.wgsl.jinja", + "bindings": ["w_w", "w_zero_point_w", "w_k_major_out", "k_major_params"], + "dispatch": { + "x": "min(ceilDiv((convKPadded * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((convKPadded * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + }, + { + "id": "im2col", + "name": "QLinearConv.Im2ColNhwc", + "shader": "qlinear-conv-im2col-nhwc.wgsl.jinja", + "derive": { + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "im2colWords": "tunables.NHWC_IM2COL_WORDS", + "channelsAligned": "convInChannels % 4 == 0", + "xVec4": "(\"vec4\" if tensorDtypes.x == \"uint8\" else \"vec4\") if convInChannels % 4 == 0 else (\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\")", + "colsVec4": "\"vec4\" if tensorDtypes.x == \"uint8\" else \"vec4\"" + }, + "bindings": ["x_im2col", "x_zero_point_w", "cols_out_vec4", "im2col_nhwc_params"], + "dispatch": { + "x": "min(ceilDiv((ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)), (tunables.WORKGROUP_SIZE)), 65535)", + "y": "ceilDiv(ceilDiv((ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)), (tunables.WORKGROUP_SIZE)), 65535)", + "z": 1 + } + }, + { + "id": "main", + "name": "QLinearConv.ExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "derive": { + "aVec4": "aScalar", + "matrixHasBias": true, + "matrixScalePerRow": "perChannelWeightQuantOk and not channelsLast", + "matrixZeroPerRow": "perChannelWeightQuantOk and not channelsLast" + }, + "bindings": ["a_cols_nhwc", "b_weight_k_major", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "gemm_params_nhwc_im2col_matrix"], + "dispatch": { "x": "ceilDiv(sgmatN, sgmatTileN)", "y": "ceilDiv(sgmatM, sgmatTileM)" } + } + ] + }, + { + "id": "precast_f16_pointwise_nchw_tensor", + "priority": 40, + "when": ["precast_pointwise_nchw_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], + "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 9 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f16\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": false, + "matrixScalePerRow": false, + "matrixZeroPerRow": false + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": false, + "sampleChannelsLast": false + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_plain"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_params"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f16_pointwise_nchw_tensor", + "priority": 28, + "when": ["precast_pointwise_nchw_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": ["shader-f16"] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f16\"", + "scalar": "\"f16\"", + "matrixHasBias": false, + "matrixScalePerRow": false, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": false, + "sampleChannelsLast": false + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_plain"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f16_pointwise_nchw_row", + "priority": 40, + "when": ["precast_pointwise_nchw_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], + "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 8 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f16\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": false, + "matrixScalePerRow": true, + "matrixZeroPerRow": true + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": true, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": false, + "sampleChannelsLast": false + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_plain"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_params"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f16_pointwise_nchw_row", + "priority": 28, + "when": ["precast_pointwise_nchw_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": ["shader-f16"] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f16\"", + "scalar": "\"f16\"", + "matrixHasBias": false, + "matrixScalePerRow": true, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": true, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": false, + "sampleChannelsLast": false + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_plain"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f16_im2col_nchw_tensor", + "priority": 40, + "when": ["precast_im2col_nchw_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], + "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 9 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f16\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": false, + "matrixScalePerRow": false, + "matrixZeroPerRow": false + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": true, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": false, + "sampleChannelsLast": false + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_params"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f16_im2col_nchw_tensor", + "priority": 28, + "when": ["precast_im2col_nchw_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": ["shader-f16"] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f16\"", + "scalar": "\"f16\"", + "matrixHasBias": false, + "matrixScalePerRow": false, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": true, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": false, + "sampleChannelsLast": false + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f16_im2col_nchw_row", + "priority": 40, + "when": ["precast_im2col_nchw_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], + "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 8 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f16\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": false, + "matrixScalePerRow": true, + "matrixZeroPerRow": true + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": true, + "prepareSpatialB": true, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": false, + "sampleChannelsLast": false + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_params"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f16_im2col_nchw_row", + "priority": 28, + "when": ["precast_im2col_nchw_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": ["shader-f16"] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f16\"", + "scalar": "\"f16\"", + "matrixHasBias": false, + "matrixScalePerRow": true, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": true, + "prepareSpatialB": true, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": false, + "sampleChannelsLast": false + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f16_im2col_nchw_bias_tensor", + "priority": 40, + "when": ["precast_im2col_nchw_bias_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], + "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 10 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f16\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": true, + "matrixScalePerRow": false, + "matrixZeroPerRow": false + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": true, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": false, + "sampleChannelsLast": false + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_params"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f16_im2col_nchw_bias_tensor", + "priority": 28, + "when": ["precast_im2col_nchw_bias_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": ["shader-f16"] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f16\"", + "scalar": "\"f16\"", + "matrixHasBias": true, + "matrixScalePerRow": false, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": true, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": false, + "sampleChannelsLast": false + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f16_im2col_nchw_bias_row", + "priority": 40, + "when": ["precast_im2col_nchw_bias_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], + "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 9 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f16\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": true, + "matrixScalePerRow": true, + "matrixZeroPerRow": true + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": true, + "prepareSpatialB": true, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": false, + "sampleChannelsLast": false + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_params"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f16_im2col_nchw_bias_row", + "priority": 28, + "when": ["precast_im2col_nchw_bias_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": ["shader-f16"] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f16\"", + "scalar": "\"f16\"", + "matrixHasBias": true, + "matrixScalePerRow": true, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": true, + "prepareSpatialB": true, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": false, + "sampleChannelsLast": false + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f16_pointwise_nhwc_tensor", + "priority": 40, + "when": ["precast_pointwise_nhwc_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], + "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 9 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f16\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": false, + "matrixScalePerRow": false, + "matrixZeroPerRow": false + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_nhwc"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f16_pointwise_nhwc_tensor", + "priority": 28, + "when": ["precast_pointwise_nhwc_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": ["shader-f16"] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f16\"", + "scalar": "\"f16\"", + "matrixHasBias": false, + "matrixScalePerRow": false, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f16_pointwise_nhwc_column", + "priority": 40, + "when": ["precast_pointwise_nhwc_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], + "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 8 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f16\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": false, + "matrixScalePerRow": false, + "matrixZeroPerRow": false + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "a_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_nhwc"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f16_pointwise_nhwc_column", + "priority": 28, + "when": ["precast_pointwise_nhwc_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": ["shader-f16"] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f16\"", + "scalar": "\"f16\"", + "matrixHasBias": false, + "matrixScalePerRow": false, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f16_pointwise_nhwc_bias_tensor", + "priority": 40, + "when": ["precast_pointwise_nhwc_bias_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], + "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 10 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f16\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": true, + "matrixScalePerRow": false, + "matrixZeroPerRow": false + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_nhwc"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f16_pointwise_nhwc_bias_tensor", + "priority": 28, + "when": ["precast_pointwise_nhwc_bias_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": ["shader-f16"] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f16\"", + "scalar": "\"f16\"", + "matrixHasBias": true, + "matrixScalePerRow": false, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f16_pointwise_nhwc_bias_column", + "priority": 40, + "when": ["precast_pointwise_nhwc_bias_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], + "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 9 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f16\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": true, + "matrixScalePerRow": false, + "matrixZeroPerRow": false + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "a_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_nhwc"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f16_pointwise_nhwc_bias_column", + "priority": 28, + "when": ["precast_pointwise_nhwc_bias_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": ["shader-f16"] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f16\"", + "scalar": "\"f16\"", + "matrixHasBias": true, + "matrixScalePerRow": false, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f16_im2col_nhwc_tensor", + "priority": 40, + "when": ["precast_im2col_nhwc_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], + "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 9 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f16\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": false, + "matrixScalePerRow": false, + "matrixZeroPerRow": false + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": true, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_nhwc"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f16_im2col_nhwc_tensor", + "priority": 28, + "when": ["precast_im2col_nhwc_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": ["shader-f16"] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f16\"", + "scalar": "\"f16\"", + "matrixHasBias": false, + "matrixScalePerRow": false, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": true, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f16_im2col_nhwc_column", + "priority": 40, + "when": ["precast_im2col_nhwc_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], + "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 8 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f16\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": false, + "matrixScalePerRow": false, + "matrixZeroPerRow": false + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": true, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "a_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_nhwc"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f16_im2col_nhwc_column", + "priority": 28, + "when": ["precast_im2col_nhwc_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": ["shader-f16"] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f16\"", + "scalar": "\"f16\"", + "matrixHasBias": false, + "matrixScalePerRow": false, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": true, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f16_im2col_nhwc_bias_tensor", + "priority": 40, + "when": ["precast_im2col_nhwc_bias_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], + "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 10 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f16\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": true, + "matrixScalePerRow": false, + "matrixZeroPerRow": false + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": true, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_nhwc"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f16_im2col_nhwc_bias_tensor", + "priority": 28, + "when": ["precast_im2col_nhwc_bias_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": ["shader-f16"] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f16\"", + "scalar": "\"f16\"", + "matrixHasBias": true, + "matrixScalePerRow": false, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": true, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f16_im2col_nhwc_bias_column", + "priority": 40, + "when": ["precast_im2col_nhwc_bias_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], + "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 9 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f16\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": true, + "matrixScalePerRow": false, + "matrixZeroPerRow": false + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": true, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "a_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_nhwc"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f16_im2col_nhwc_bias_column", + "priority": 28, + "when": ["precast_im2col_nhwc_bias_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": ["shader-f16"] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f16\"", + "scalar": "\"f16\"", + "matrixHasBias": true, + "matrixScalePerRow": false, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": true, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f32_pointwise_nchw_tensor", + "priority": 40, + "when": ["precast_pointwise_nchw_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix"], + "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 9 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f32\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": false, + "matrixScalePerRow": false, + "matrixZeroPerRow": false + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": false, + "sampleChannelsLast": false + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_plain"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_params"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f32_pointwise_nchw_tensor", + "priority": 28, + "when": ["precast_pointwise_nchw_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": [] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f32\"", + "scalar": "\"f32\"", + "matrixHasBias": false, + "matrixScalePerRow": false, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": false, + "sampleChannelsLast": false + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_plain"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f32_pointwise_nchw_row", + "priority": 40, + "when": ["precast_pointwise_nchw_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix"], + "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 8 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f32\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": false, + "matrixScalePerRow": true, + "matrixZeroPerRow": true + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": true, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": false, + "sampleChannelsLast": false + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_plain"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_params"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f32_pointwise_nchw_row", + "priority": 28, + "when": ["precast_pointwise_nchw_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": [] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f32\"", + "scalar": "\"f32\"", + "matrixHasBias": false, + "matrixScalePerRow": true, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": true, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": false, + "sampleChannelsLast": false + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_plain"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f32_im2col_nchw_tensor", + "priority": 40, + "when": ["precast_im2col_nchw_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix"], + "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 9 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f32\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": false, + "matrixScalePerRow": false, + "matrixZeroPerRow": false + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": true, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": false, + "sampleChannelsLast": false + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_params"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f32_im2col_nchw_tensor", + "priority": 28, + "when": ["precast_im2col_nchw_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": [] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f32\"", + "scalar": "\"f32\"", + "matrixHasBias": false, + "matrixScalePerRow": false, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": true, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": false, + "sampleChannelsLast": false + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f32_im2col_nchw_row", + "priority": 40, + "when": ["precast_im2col_nchw_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix"], + "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 8 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f32\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": false, + "matrixScalePerRow": true, + "matrixZeroPerRow": true + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": true, + "prepareSpatialB": true, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": false, + "sampleChannelsLast": false + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_params"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f32_im2col_nchw_row", + "priority": 28, + "when": ["precast_im2col_nchw_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": [] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f32\"", + "scalar": "\"f32\"", + "matrixHasBias": false, + "matrixScalePerRow": true, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": true, + "prepareSpatialB": true, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": false, + "sampleChannelsLast": false + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f32_im2col_nchw_bias_tensor", + "priority": 40, + "when": ["precast_im2col_nchw_bias_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix"], + "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 10 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f32\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": true, + "matrixScalePerRow": false, + "matrixZeroPerRow": false + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": true, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": false, + "sampleChannelsLast": false + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_params"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f32_im2col_nchw_bias_tensor", + "priority": 28, + "when": ["precast_im2col_nchw_bias_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": [] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f32\"", + "scalar": "\"f32\"", + "matrixHasBias": true, + "matrixScalePerRow": false, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": true, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": false, + "sampleChannelsLast": false + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f32_im2col_nchw_bias_row", + "priority": 40, + "when": ["precast_im2col_nchw_bias_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix"], + "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 9 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f32\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": true, + "matrixScalePerRow": true, + "matrixZeroPerRow": true + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": true, + "prepareSpatialB": true, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": false, + "sampleChannelsLast": false + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_params"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f32_im2col_nchw_bias_row", + "priority": 28, + "when": ["precast_im2col_nchw_bias_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": [] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f32\"", + "scalar": "\"f32\"", + "matrixHasBias": true, + "matrixScalePerRow": true, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": true, + "prepareSpatialB": true, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": false, + "sampleChannelsLast": false + }, + "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f32_pointwise_nhwc_tensor", + "priority": 40, + "when": ["precast_pointwise_nhwc_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix"], + "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 9 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f32\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": false, + "matrixScalePerRow": false, + "matrixZeroPerRow": false + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_nhwc"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f32_pointwise_nhwc_tensor", + "priority": 28, + "when": ["precast_pointwise_nhwc_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": [] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f32\"", + "scalar": "\"f32\"", + "matrixHasBias": false, + "matrixScalePerRow": false, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f32_pointwise_nhwc_column", + "priority": 40, + "when": ["precast_pointwise_nhwc_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix"], + "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 8 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f32\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": false, + "matrixScalePerRow": false, + "matrixZeroPerRow": false + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "a_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_nhwc"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f32_pointwise_nhwc_column", + "priority": 28, + "when": ["precast_pointwise_nhwc_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": [] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f32\"", + "scalar": "\"f32\"", + "matrixHasBias": false, + "matrixScalePerRow": false, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f32_pointwise_nhwc_bias_tensor", + "priority": 40, + "when": ["precast_pointwise_nhwc_bias_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix"], + "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 10 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f32\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": true, + "matrixScalePerRow": false, + "matrixZeroPerRow": false + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_nhwc"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f32_pointwise_nhwc_bias_tensor", + "priority": 28, + "when": ["precast_pointwise_nhwc_bias_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": [] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f32\"", + "scalar": "\"f32\"", + "matrixHasBias": true, + "matrixScalePerRow": false, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f32_pointwise_nhwc_bias_column", + "priority": 40, + "when": ["precast_pointwise_nhwc_bias_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix"], + "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 9 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f32\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": true, + "matrixScalePerRow": false, + "matrixZeroPerRow": false + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "a_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_nhwc"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f32_pointwise_nhwc_bias_column", + "priority": 28, + "when": ["precast_pointwise_nhwc_bias_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": [] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f32\"", + "scalar": "\"f32\"", + "matrixHasBias": true, + "matrixScalePerRow": false, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": false, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f32_im2col_nhwc_tensor", + "priority": 40, + "when": ["precast_im2col_nhwc_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix"], + "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 9 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f32\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": false, + "matrixScalePerRow": false, + "matrixZeroPerRow": false + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": true, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_nhwc"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f32_im2col_nhwc_tensor", + "priority": 28, + "when": ["precast_im2col_nhwc_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": [] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f32\"", + "scalar": "\"f32\"", + "matrixHasBias": false, + "matrixScalePerRow": false, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": true, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f32_im2col_nhwc_column", + "priority": 40, + "when": ["precast_im2col_nhwc_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix"], + "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 8 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f32\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": false, + "matrixScalePerRow": false, + "matrixZeroPerRow": false + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": true, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "a_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_nhwc"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f32_im2col_nhwc_column", + "priority": 28, + "when": ["precast_im2col_nhwc_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": [] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f32\"", + "scalar": "\"f32\"", + "matrixHasBias": false, + "matrixScalePerRow": false, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": true, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f32_im2col_nhwc_bias_tensor", + "priority": 40, + "when": ["precast_im2col_nhwc_bias_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix"], + "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 10 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f32\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": true, + "matrixScalePerRow": false, + "matrixZeroPerRow": false + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": true, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_nhwc"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f32_im2col_nhwc_bias_tensor", + "priority": 28, + "when": ["precast_im2col_nhwc_bias_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": [] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f32\"", + "scalar": "\"f32\"", + "matrixHasBias": true, + "matrixScalePerRow": false, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": true, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "precast_f32_im2col_nhwc_bias_column", + "priority": 40, + "when": ["precast_im2col_nhwc_bias_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], + "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], + "requires": { + "features": ["subgroups", "chromium-experimental-subgroup-matrix"], + "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], + "limits": { "maxStorageBuffersPerShaderStage": 9 } + }, + "derive": { + "aVec4": "aScalar", + "precastOperands": true, + "precastScalar": "\"f32\"", + "sgmatTileM": "precastTileM", + "sgmatTileN": "precastTileN", + "sgmatTileK": "precastTileK", + "sgmatRowGroups": "precastSubgroups", + "sgmatSubgroups": "precastSubgroups", + "sgmatWorkgroup": "precastWorkgroup", + "matrixHasBias": true, + "matrixScalePerRow": false, + "matrixZeroPerRow": false + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": true, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactMatrix", + "shader": "quant-exact-matrix.wgsl.jinja", + "bindings": ["precast_a", "precast_b", "a_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_nhwc"], + "dispatch": { + "x": "ceilDiv(sgmatN, precastTileN)", + "y": "ceilDiv(sgmatM, precastTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + }, + { + "id": "portable_f32_im2col_nhwc_bias_column", + "priority": 28, + "when": ["precast_im2col_nhwc_bias_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], + "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], + "requires": { "features": [] }, + "derive": { + "aVec4": "aScalar", + "precastScalar": "\"f32\"", + "scalar": "\"f32\"", + "matrixHasBias": true, + "matrixScalePerRow": false, + "M": "sgmatM", + "N": "sgmatN", + "K": "preparedK", + "kTile": "portableTileK", + "workgroupCols": "portableWorkgroupCols", + "workgroupRows": "portableWorkgroupRows", + "blockRows": "portableTileM", + "blockCols": "portableTileN", + "integerPartialK": "portablePartialK", + "hasBias": false, + "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" + }, + "intermediates": [ + { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, + { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } + ], + "passes": [ + { + "id": "prepare", + "name": "QLinearConv.PrepareExact", + "shader": "quant-exact-prepare.wgsl.jinja", + "derive": { + "prepareZeroPerRow": false, + "prepareSpatialB": false, + "spatialRankSpec": 2, + "kernelHSpec": "kernelHeight", + "kernelWSpec": "kernelWidth", + "strideHSpec": "strideH", + "strideWSpec": "strideW", + "dilationHSpec": "dilationH", + "dilationWSpec": "dilationW", + "padTopSpec": "effectivePadTop", + "padLeftSpec": "effectivePadLeft", + "prepareSpatialA": true, + "prepareTransposeB": true, + "sampleChannelsLast": true + }, + "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], + "dispatch": { + "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", + "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" + } + }, + { + "id": "main", + "name": "QLinearConv.PreparedExactPortable", + "shader": "quant-exact-portable.wgsl.jinja", + "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], + "dispatch": { + "x": "ceilDiv(sgmatN, portableTileN)", + "y": "ceilDiv(sgmatM, portableTileM)", + "z": "1 if channelsLast else dim(shapes.x, 0)" + } + } + ] + } + ] +}