{ "domain": "com.microsoft", "name": "QLinearConv", "sinceVersion": 1, "inputs": { "x": { "dtype": "TX" }, "x_scale": { "dtype": "TF" }, "x_zero_point": { "dtype": "TX" }, "w": { "dtype": "TW" }, "w_scale": { "dtype": "TF" }, "w_zero_point": { "dtype": "TW" }, "y_scale": { "dtype": "TF" }, "y_zero_point": { "dtype": "TY" }, "B": { "dtype": "int32", "rank": 1, "optional": true } }, "outputs": { "y": { "dtype": "TY", "rank": "ranks.x", "shape": "([dim(shapes.x, 0), expectedOutputWidth, dim(shapes.w, 0)] if ranks.x == 3 else ([dim(shapes.x, 0), expectedOutputHeight, expectedOutputWidth, dim(shapes.w, 0)] if ranks.x == 4 else [dim(shapes.x, 0), expectedOutputDepth, expectedOutputHeight, expectedOutputWidth, dim(shapes.w, 0)])) if channelsLast else ([dim(shapes.x, 0), dim(shapes.w, 0), expectedOutputWidth] if ranks.x == 3 else ([dim(shapes.x, 0), dim(shapes.w, 0), expectedOutputHeight, expectedOutputWidth] if ranks.x == 4 else [dim(shapes.x, 0), dim(shapes.w, 0), expectedOutputDepth, expectedOutputHeight, expectedOutputWidth]))" } }, "attributes": { "auto_pad": { "default": "NOTSET" }, "channels_last": { "default": 0 }, "group": { "default": 1 }, "dilations": {}, "kernel_shape": {}, "pads": {}, "strides": {} }, "attributeConstraints": { "auto_pad": { "values": ["NOTSET", "SAME_UPPER", "SAME_LOWER", "VALID"] } }, "typeConstraints": { "TX": ["uint8", "int8"], "TW": ["uint8", "int8"], "TY": ["uint8", "int8"], "TF": ["float32"] }, "tunables": { "WORKGROUP_SIZE": { "default": 256 }, "NHWC_OW_TILE": { "default": 4 }, "NHWC_IM2COL_WORDS": { "default": 2 }, "EXACT_MATRIX_MIN_INVOCATIONS": { "default": 10240 }, "NHWC_BIAS_TARGET_WORKGROUPS": { "default": 64 } }, "derive": { "channelsLast": "attrs.channels_last != 0", "convInChannels": "dim(shapes.x, ranks.x - 1) if channelsLast else dim(shapes.x, 1)", "convOutChannels": "dim(shapes.w, 0)", "inputDepth": "(dim(shapes.x, 1) if channelsLast else dim(shapes.x, 2)) if ranks.x == 5 else 1", "inputHeight": "((dim(shapes.x, 2) if channelsLast else dim(shapes.x, 3)) if ranks.x == 5 else ((dim(shapes.x, 1) if channelsLast else dim(shapes.x, 2)) if ranks.x == 4 else 1))", "inputWidth": "(dim(shapes.x, ranks.x - 2) if channelsLast else dim(shapes.x, ranks.x - 1)) if ranks.x >= 3 else 1", "outputDepth": "(dim(shapes.y, 1) if channelsLast else dim(shapes.y, 2)) if ranks.y == 5 else 1", "outputHeight": "((dim(shapes.y, 2) if channelsLast else dim(shapes.y, 3)) if ranks.y == 5 else ((dim(shapes.y, 1) if channelsLast else dim(shapes.y, 2)) if ranks.y == 4 else 1))", "outputWidth": "(dim(shapes.y, ranks.y - 2) if channelsLast else dim(shapes.y, ranks.y - 1)) if ranks.y >= 3 else 1", "kernelDepth": "dim(shapes.w, 2) if ranks.w == 5 else 1", "kernelHeight": "dim(shapes.w, 3) if ranks.w == 5 else (dim(shapes.w, 2) if ranks.w == 4 else 1)", "kernelWidth": "dim(shapes.w, ranks.w - 1) if ranks.w >= 3 else 1", "spatialRank": "ranks.w - 2", "outSpatial": "outputDepth * outputHeight * outputWidth", "convK": "dim(shapes.w, 1) * kernelDepth * kernelHeight * kernelWidth", "dp4aCandidateKWords": 128, "dp4aAppleCompact": "device.adapterInfo.vendor == \"apple\" and device.features.has(\"subgroups\") and not channelsLast and not present.B and attrs.group == 1 and (kernelHeight > 1 or kernelWidth > 1) and convOutChannels >= outSpatial and convK >= 4 * dp4aCandidateKWords and ceilDiv(convOutChannels, 4) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(outSpatial, 4) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "dp4aCompactKWords": "dp4aCandidateKWords if dp4aAppleCompact else 32", "sgmatM": "dim(shapes.x, 0) * outSpatial if channelsLast else convOutChannels", "sgmatN": "convOutChannels if channelsLast else outSpatial", "sgmatTileRows": "sgmatM", "precastRows": "sgmatM", "precastContraction": "convK", "precastCols": "sgmatN", "precastBatches": "1 if channelsLast else dim(shapes.x, 0)", "portableWorkgroupCols": 8, "kernelShapeLengthOk": "not has(attrs, \"kernel_shape\") or (attrs.kernel_shape | length) == spatialRank", "stridesLengthOk": "not has(attrs, \"strides\") or (attrs.strides | length) == spatialRank", "dilationsLengthOk": "not has(attrs, \"dilations\") or (attrs.dilations | length) == spatialRank", "padsLengthOk": "not has(attrs, \"pads\") or (attrs.pads | length) == 2 * spatialRank", "kernelD": "attrs.kernel_shape[0] if kernelShapeLengthOk and has(attrs, \"kernel_shape\") and spatialRank == 3 else 1", "kernelH": "attrs.kernel_shape[spatialRank - 2] if kernelShapeLengthOk and has(attrs, \"kernel_shape\") and spatialRank >= 2 else 1", "kernelW": "attrs.kernel_shape[spatialRank - 1] if kernelShapeLengthOk and has(attrs, \"kernel_shape\") and spatialRank >= 1 else 1", "strideD": "attrs.strides[0] if stridesLengthOk and has(attrs, \"strides\") and spatialRank == 3 else 1", "strideH": "attrs.strides[spatialRank - 2] if stridesLengthOk and has(attrs, \"strides\") and spatialRank >= 2 else 1", "strideW": "attrs.strides[spatialRank - 1] if stridesLengthOk and has(attrs, \"strides\") and spatialRank >= 1 else 1", "dilationD": "attrs.dilations[0] if dilationsLengthOk and has(attrs, \"dilations\") and spatialRank == 3 else 1", "dilationH": "attrs.dilations[spatialRank - 2] if dilationsLengthOk and has(attrs, \"dilations\") and spatialRank >= 2 else 1", "dilationW": "attrs.dilations[spatialRank - 1] if dilationsLengthOk and has(attrs, \"dilations\") and spatialRank >= 1 else 1", "padFront": "attrs.pads[0] if padsLengthOk and has(attrs, \"pads\") and spatialRank == 3 else 0", "padTop": "attrs.pads[spatialRank - 2] if padsLengthOk and has(attrs, \"pads\") and spatialRank >= 2 else 0", "padLeft": "attrs.pads[spatialRank - 1] if padsLengthOk and has(attrs, \"pads\") and spatialRank >= 1 else 0", "padBack": "attrs.pads[spatialRank] if padsLengthOk and has(attrs, \"pads\") and spatialRank == 3 else 0", "padBottom": "attrs.pads[2 * spatialRank - 2] if padsLengthOk and has(attrs, \"pads\") and spatialRank >= 2 else 0", "padRight": "attrs.pads[2 * spatialRank - 1] if padsLengthOk and has(attrs, \"pads\") and spatialRank >= 1 else 0", "autoPadSame": "attrs.auto_pad == \"SAME_UPPER\" or attrs.auto_pad == \"SAME_LOWER\"", "autoPadValid": "attrs.auto_pad == \"VALID\"", "samePadDepth": "max(0, (outputDepth - 1) * strideD + (kernelDepth - 1) * dilationD + 1 - inputDepth)", "samePadHeight": "max(0, (outputHeight - 1) * strideH + (kernelHeight - 1) * dilationH + 1 - inputHeight)", "samePadWidth": "max(0, (outputWidth - 1) * strideW + (kernelWidth - 1) * dilationW + 1 - inputWidth)", "samePadFront": "floor(samePadDepth / 2) if attrs.auto_pad == \"SAME_UPPER\" else samePadDepth - floor(samePadDepth / 2)", "samePadTop": "floor(samePadHeight / 2) if attrs.auto_pad == \"SAME_UPPER\" else samePadHeight - floor(samePadHeight / 2)", "samePadLeft": "floor(samePadWidth / 2) if attrs.auto_pad == \"SAME_UPPER\" else samePadWidth - floor(samePadWidth / 2)", "effectivePadFront": "samePadFront if autoPadSame else (0 if autoPadValid else padFront)", "effectivePadTop": "samePadTop if autoPadSame else (0 if autoPadValid else padTop)", "effectivePadLeft": "samePadLeft if autoPadSame else (0 if autoPadValid else padLeft)", "expectedOutputDepth": "ceil(inputDepth / strideD) if autoPadSame else floor((inputDepth + (0 if autoPadValid else padFront + padBack) - ((kernelDepth - 1) * dilationD + 1)) / strideD) + 1", "expectedOutputHeight": "ceil(inputHeight / strideH) if autoPadSame else floor((inputHeight + (0 if autoPadValid else padTop + padBottom) - ((kernelHeight - 1) * dilationH + 1)) / strideH) + 1", "expectedOutputWidth": "ceil(inputWidth / strideW) if autoPadSame else floor((inputWidth + (0 if autoPadValid else padLeft + padRight) - ((kernelWidth - 1) * dilationW + 1)) / strideW) + 1", "wave32Adapter": "has(device.adapterInfo, \"subgroupMinSize\") and has(device.adapterInfo, \"subgroupMaxSize\") and device.adapterInfo.subgroupMinSize == 32 and device.adapterInfo.subgroupMaxSize == 32", "canPinSubgroupSize32": "device.features.has(\"subgroups\") and device.features.has(\"subgroup-size-control\") and has(device.adapterInfo, \"subgroupMinSize\") and has(device.adapterInfo, \"subgroupMaxSize\") and device.adapterInfo.subgroupMinSize <= 32 and device.adapterInfo.subgroupMaxSize >= 32", "pinSubgroupSize32": "canPinSubgroupSize32 and not wave32Adapter", "wave32Effective": "wave32Adapter or pinSubgroupSize32", "deviceWorkgroupCap": "min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)", "sgmatTileN": "64", "sgmatTileK": "32", "sgmatColumnGroups": "2", "sgmatStorageRowGroups": "floor((device.limits.maxComputeWorkgroupStorageSize - sgmatTileN * sgmatTileK * 4) / (16 * sgmatTileK * 4))", "sgmatRowGroups": "max(1, min(4, floor(deviceWorkgroupCap / (32 * sgmatColumnGroups)), sgmatStorageRowGroups, max(floor(sgmatStorageRowGroups / 2), floor(sgmatTileRows / 16))))", "sgmatTileM": "16 * sgmatRowGroups", "sgmatSubgroups": "sgmatRowGroups * sgmatColumnGroups", "sgmatWorkgroup": "32 * sgmatSubgroups", "sgmatStagingBytes": "(sgmatTileM + sgmatTileN) * sgmatTileK * 4", "sgmatScratchBytes": "sgmatSubgroups * 4 * 64 * 4", "sgmatReuseStaging": "sgmatStagingBytes + sgmatScratchBytes > device.limits.maxComputeWorkgroupStorageSize", "sgmatSharedBytes": "sgmatStagingBytes + (0 if sgmatReuseStaging else sgmatScratchBytes)", "sgmatDeviceOk": "sgmatWorkgroup <= deviceWorkgroupCap and sgmatSharedBytes <= device.limits.maxComputeWorkgroupStorageSize", "precastSubgroups": "max(1, min(4, ceilDiv(precastRows, 16), floor(deviceWorkgroupCap / 32), floor(device.limits.maxComputeWorkgroupStorageSize / (8 * 64 * 4))))", "precastTileM": "16 * precastSubgroups", "precastTileN": "4 * 8", "precastTileK": "4 * 8", "precastWorkgroup": "32 * precastSubgroups", "preparedK": "ceilDiv(precastContraction, precastTileK) * precastTileK", "preparedN": "ceilDiv(precastCols, precastTileN) * precastTileN", "preparedElements": "ceilDiv(precastRows, precastTileM) * precastTileM * preparedK", "preparedBElements": "precastBatches * preparedK * preparedN", "prepareWorkgroup": "min(256, deviceWorkgroupCap)", "prepareTotalElements": "preparedElements + preparedBElements", "prepareGroups": "ceilDiv(prepareTotalElements, 4 * prepareWorkgroup)", "prepareDispatchElements": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535)) * min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535)) * prepareWorkgroup * 4", "precastResourcesOk": "deviceWorkgroupCap >= precastWorkgroup and precastSubgroups * 8 * 64 * 4 <= device.limits.maxComputeWorkgroupStorageSize", "precastDispatchOk": "prepareTotalElements <= 4294967295 and prepareDispatchElements <= 4294967296 and ceilDiv(precastCols, precastTileN) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(precastRows, precastTileM) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and prepareGroups <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) * min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "precastF32BuffersOk": "preparedElements * 4 <= device.limits.maxStorageBufferBindingSize and preparedElements * 4 <= device.limits.maxBufferSize and preparedBElements * 4 <= device.limits.maxStorageBufferBindingSize and preparedBElements * 4 <= device.limits.maxBufferSize", "precastF16BuffersOk": "preparedElements * 2 <= device.limits.maxStorageBufferBindingSize and preparedElements * 2 <= device.limits.maxBufferSize and preparedBElements * 2 <= device.limits.maxStorageBufferBindingSize and preparedBElements * 2 <= device.limits.maxBufferSize", "portableWorkgroupRows": "pow2ceil(min(16, max(4, pow2ceil(ceilDiv(precastRows, 4))), floor(deviceWorkgroupCap / portableWorkgroupCols), device.limits.maxComputeWorkgroupSizeY) + 1) / 2", "portableTileM": "4 * portableWorkgroupRows", "portableTileN": "4 * portableWorkgroupCols", "portableTileK": 32, "portablePartialK": "floor(16777216 / (255 * 255) / portableTileK) * portableTileK", "portableGeometryOk": "portableWorkgroupRows >= 4 and portableWorkgroupCols <= device.limits.maxComputeWorkgroupSizeX and portablePartialK >= portableTileK and ceilDiv(precastRows, portableTileM) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "spatialAttributeLengthsOk": "kernelShapeLengthOk and stridesLengthOk and dilationsLengthOk and padsLengthOk", "kernelShapeMatchesWeights": "not has(attrs, \"kernel_shape\") or (kernelW == kernelWidth and (spatialRank < 2 or kernelH == kernelHeight) and (spatialRank < 3 or kernelD == kernelDepth))", "kernelExtentsOk": "kernelWidth >= 1 and (spatialRank < 2 or kernelHeight >= 1) and (spatialRank < 3 or kernelDepth >= 1)", "stridesValuesOk": "not has(attrs, \"strides\") or (strideD >= 1 and floor(strideD) == strideD and strideH >= 1 and floor(strideH) == strideH and strideW >= 1 and floor(strideW) == strideW)", "dilationsValuesOk": "not has(attrs, \"dilations\") or (dilationD >= 1 and floor(dilationD) == dilationD and dilationH >= 1 and floor(dilationH) == dilationH and dilationW >= 1 and floor(dilationW) == dilationW)", "padsValuesOk": "padFront >= 0 and floor(padFront) == padFront and padTop >= 0 and floor(padTop) == padTop and padLeft >= 0 and floor(padLeft) == padLeft and padBack >= 0 and floor(padBack) == padBack and padBottom >= 0 and floor(padBottom) == padBottom and padRight >= 0 and floor(padRight) == padRight", "explicitPadsOk": "attrs.auto_pad == \"NOTSET\" or not has(attrs, \"pads\")", "spatialAttributesOk": "spatialRank >= 1 and spatialRank <= 3 and spatialAttributeLengthsOk and kernelShapeMatchesWeights and kernelExtentsOk and stridesValuesOk and dilationsValuesOk and padsValuesOk and explicitPadsOk", "xyQuantContract": "(ranks.x_scale == 0 or (ranks.x_scale == 1 and dim(shapes.x_scale, 0) == 1)) and (ranks.y_scale == 0 or (ranks.y_scale == 1 and dim(shapes.y_scale, 0) == 1)) and (ranks.x_zero_point == 0 or (ranks.x_zero_point == 1 and dim(shapes.x_zero_point, 0) == 1)) and (ranks.y_zero_point == 0 or (ranks.y_zero_point == 1 and dim(shapes.y_zero_point, 0) == 1))", "rank3TensorOk": "spatialAttributesOk and ranks.x == 3 and ranks.w == 3 and ranks.y == 3", "rank5TensorOk": "spatialAttributesOk and ranks.x == 5 and ranks.w == 5 and ranks.y == 5", "scalarWeightQuantOk": "(ranks.w_scale == 0 or (ranks.w_scale == 1 and dim(shapes.w_scale, 0) == 1)) and (ranks.w_zero_point == 0 or (ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == 1))", "yScalar": "\"u32\" if tensorDtypes.y == \"uint8\" else \"i32\"", "yUnsigned": "tensorDtypes.y == \"uint8\"", "qMin": "0 if tensorDtypes.y == \"uint8\" else 0 - 128", "qMax": "255 if tensorDtypes.y == \"uint8\" else 127", "hasBZero": true, "packedFeature": "device.wgslLanguageFeatures.has(\"packed_4x8_integer_dot_product\")", "rank4TensorOk": "spatialAttributesOk and ranks.x == 4 and ranks.w == 4 and ranks.y == 4", "outputBatchChannelsOk": "dim(shapes.y, 0) == dim(shapes.x, 0) and (dim(shapes.y, ranks.y - 1) if channelsLast else dim(shapes.y, 1)) == convOutChannels", "groupChannelsOk": "attrs.group >= 1 and dim(shapes.w, 1) * attrs.group == convInChannels and convOutChannels % attrs.group == 0", "outputSpatialOk": "outputWidth == expectedOutputWidth and (spatialRank < 2 or outputHeight == expectedOutputHeight) and (spatialRank < 3 or outputDepth == expectedOutputDepth)", "perChannelWeightQuantOk": "ranks.w_scale == 1 and dim(shapes.w_scale, 0) == convOutChannels and ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == convOutChannels", "flexibleWeightQuantOk": "scalarWeightQuantOk or perChannelWeightQuantOk", "outImageSize": "1 if channelsLast else outSpatial", "convKPadded": "ceilDiv(convK, 4) * 4", "biasContractOk": "present.B and ranks.B == 1 and dim(shapes.B, 0) == convOutChannels", "gemmDispatchOk": "ceilDiv(outSpatial, 64) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(convOutChannels, 64) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(convOutChannels, 8) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(outSpatial, 8) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and dim(shapes.x, 0) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "kMajorWeightStorageOk": "convKPadded * convOutChannels * 4 <= device.limits.maxStorageBufferBindingSize", "nhwcColsStorageOk": "dim(shapes.x, 0) * outSpatial * convKPadded * 4 <= device.limits.maxStorageBufferBindingSize", "nchwColsStorageOk": "dim(shapes.x, 0) * convK * outSpatial * 4 <= device.limits.maxStorageBufferBindingSize", "aUnsigned": "tensorDtypes.x == \"uint8\" if channelsLast else tensorDtypes.w == \"uint8\"", "bUnsigned": "tensorDtypes.w == \"uint8\" if channelsLast else tensorDtypes.x == \"uint8\"", "aScalar": "\"u32\" if aUnsigned else \"i32\"", "bScalar": "\"u32\" if bUnsigned else \"i32\"", "xScalar": "\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\"", "wScalar": "\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\"", "aZeroPerRow": "perChannelWeightQuantOk and not channelsLast", "bZeroPerColumn": "perChannelWeightQuantOk and channelsLast", "requant": "(\"per_column\" if channelsLast else \"per_row\") if perChannelWeightQuantOk else \"per_tensor\"", "wZeroPerChannel": "perChannelWeightQuantOk", "sgmatDispatchOk": "ceilDiv(sgmatN, sgmatTileN) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(sgmatM, sgmatTileM) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "sgmatFullGrid": "floor(sgmatM / sgmatTileM) * floor(sgmatN / sgmatTileN) * (1 if channelsLast else dim(shapes.x, 0))", "sgmatUsefulInvocations": "sgmatFullGrid * sgmatWorkgroup * convK / (convK + 2 * sgmatTileK)", "precastFullGrid": "floor(sgmatM / precastTileM) * floor(sgmatN / precastTileN) * (1 if channelsLast else dim(shapes.x, 0))", "precastUsefulInvocations": "precastFullGrid * precastWorkgroup * convK / (convK + 2 * precastTileK)", "precastAmortizedInvocations": "precastUsefulInvocations * max(1, (convK - 2 * precastTileK) / precastTileK)", "precast_pointwise_nchw_ok": "(attrs.channels_last == 0) and (not present.B) and (rank4TensorOk) and (flexibleWeightQuantOk) and (attrs.group == 1) and (kernelHeight == 1) and (kernelWidth == 1) and (strideH == 1) and (strideW == 1) and (padTop == 0 and padBottom == 0) and (padLeft == 0 and padRight == 0) and (dim(shapes.w, 1) == convInChannels) and (outputHeight == inputHeight) and (outputWidth == inputWidth) and (gemmDispatchOk)", "precast_im2col_nchw_ok": "(attrs.channels_last == 0) and (not present.B) and (rank4TensorOk) and (flexibleWeightQuantOk) and (attrs.group == 1) and (dim(shapes.w, 1) == convInChannels) and (outputSpatialOk) and ((kernelHeight > 1 or kernelWidth > 1)) and (convK >= 64) and (convK <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)) and (nchwColsStorageOk) and (gemmDispatchOk)", "precast_im2col_nchw_bias_ok": "(attrs.channels_last == 0) and (biasContractOk) and (rank4TensorOk) and (flexibleWeightQuantOk) and (attrs.group == 1) and (dim(shapes.w, 1) == convInChannels) and (outputSpatialOk) and (convK <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)) and (nchwColsStorageOk) and (gemmDispatchOk)", "precast_pointwise_nhwc_ok": "(attrs.channels_last != 0) and (not present.B) and (rank4TensorOk) and (flexibleWeightQuantOk) and (attrs.group == 1) and (dim(shapes.w, 1) == convInChannels) and (kernelHeight == 1) and (kernelWidth == 1) and (strideH == 1) and (strideW == 1) and (padTop == 0 and padBottom == 0) and (padLeft == 0 and padRight == 0) and (outputHeight == inputHeight) and (outputWidth == inputWidth) and (kMajorWeightStorageOk) and (gemmDispatchOk)", "precast_pointwise_nhwc_bias_ok": "(attrs.channels_last != 0) and (biasContractOk) and (rank4TensorOk) and (flexibleWeightQuantOk) and (attrs.group == 1) and (dim(shapes.w, 1) == convInChannels) and (kernelHeight == 1) and (kernelWidth == 1) and (strideH == 1) and (strideW == 1) and (padTop == 0 and padBottom == 0) and (padLeft == 0 and padRight == 0) and (outputHeight == inputHeight) and (outputWidth == inputWidth) and (kMajorWeightStorageOk) and (gemmDispatchOk)", "precast_im2col_nhwc_ok": "(attrs.channels_last != 0) and (not present.B) and (rank4TensorOk) and (flexibleWeightQuantOk) and (attrs.group == 1) and (dim(shapes.w, 1) == convInChannels) and ((kernelHeight > 1 or kernelWidth > 1)) and (outputSpatialOk) and (nhwcColsStorageOk) and (kMajorWeightStorageOk) and (gemmDispatchOk)", "precast_im2col_nhwc_bias_ok": "(attrs.channels_last != 0) and (biasContractOk) and (rank4TensorOk) and (flexibleWeightQuantOk) and (attrs.group == 1) and (dim(shapes.w, 1) == convInChannels) and ((kernelHeight > 1 or kernelWidth > 1)) and (outputSpatialOk) and (nhwcColsStorageOk) and (kMajorWeightStorageOk) and (gemmDispatchOk)", "portableLoadsOk": "portableTileM * portableTileK % (4 * portableWorkgroupRows * portableWorkgroupCols) == 0 and portableTileN * portableTileK % (4 * portableWorkgroupRows * portableWorkgroupCols) == 0", "nhwcOc4UnitColumnThreads": "dim(shapes.y, 0) * outputHeight * outputWidth * ceilDiv(convOutChannels, 4)", "nhwcOc4ColumnTile": "min(tunables.NHWC_OW_TILE, max(1, floor(nhwcOc4UnitColumnThreads / (tunables.WORKGROUP_SIZE * tunables.NHWC_BIAS_TARGET_WORKGROUPS)))) if present.B else tunables.NHWC_OW_TILE", "nhwcOc4Threads": "dim(shapes.y, 0) * outputHeight * ceilDiv(outputWidth, nhwcOc4ColumnTile) * ceilDiv(convOutChannels, 4)", "nhwcOc4Workgroups": "ceilDiv(nhwcOc4Threads, tunables.WORKGROUP_SIZE)", "nhwcOc4DispatchOk": "min(nhwcOc4Workgroups, DISPATCH_FOLD_WIDTH) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(nhwcOc4Workgroups, DISPATCH_FOLD_WIDTH) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "bScalePerColumn": "perChannelWeightQuantOk and channelsLast", "matrixBatchB": "not channelsLast", "matrixBiasPerRow": "not channelsLast", "matrixSubgroupId": true, "prepareBatchB": "not channelsLast", "requantize": true, "wUnsigned": "tensorDtypes.w == \"uint8\"", "xUnsigned": "tensorDtypes.x == \"uint8\"" }, "when": ["outputBatchChannelsOk", "xyQuantContract"], "bindings": { "x_w": { "name": "x", "elementType": "$xScalar" }, "x_zero_point_w": { "name": "x_zero_point", "elementType": "$xScalar", "length": 1 }, "w_w": { "name": "w", "elementType": "$wScalar" }, "w_zero_point_w": { "name": "w_zero_point", "elementType": "$wScalar" }, "x_scale": { "elementType": "f32", "length": 1 }, "w_scale": { "elementType": "f32" }, "y_scale": { "elementType": "f32", "length": 1 }, "y_zero_point_w": { "name": "y_zero_point", "elementType": "$yScalar", "length": 1 }, "y_w": { "name": "y", "elementType": "$yScalar" }, "B": { "elementType": "i32" }, "accum_out": { "scratch": "accum", "name": "y", "elementType": "i32" }, "accum_in": { "scratch": "accum", "name": "accum", "buffer": "read-only-storage", "elementType": "i32" }, "accum_params_2d": { "name": "params", "struct": [ { "name": "inChannels", "type": "u32", "value": "convInChannels" }, { "name": "inH", "type": "u32", "value": "inputHeight" }, { "name": "inW", "type": "u32", "value": "inputWidth" }, { "name": "outChannels", "type": "u32", "value": "convOutChannels" }, { "name": "weightInChannels", "type": "u32", "value": "dim(shapes.w, 1)" }, { "name": "inChannelsPerGroup", "type": "u32", "value": "convInChannels / attrs.group" }, { "name": "outChannelsPerGroup", "type": "u32", "value": "convOutChannels / attrs.group" }, { "name": "kernelH", "type": "u32", "value": "kernelHeight" }, { "name": "kernelW", "type": "u32", "value": "kernelWidth" }, { "name": "outH", "type": "u32", "value": "outputHeight" }, { "name": "outW", "type": "u32", "value": "outputWidth" }, { "name": "strideH", "type": "u32", "value": "strideH" }, { "name": "strideW", "type": "u32", "value": "strideW" }, { "name": "dilationH", "type": "u32", "value": "dilationH" }, { "name": "dilationW", "type": "u32", "value": "dilationW" }, { "name": "padH", "type": "i32", "value": "effectivePadTop" }, { "name": "padW", "type": "i32", "value": "effectivePadLeft" }, { "name": "count", "type": "u32", "value": "numel(shapes.y)" } ] }, "accum_params_3d": { "name": "params", "struct": [ { "name": "inChannels", "type": "u32", "value": "convInChannels" }, { "name": "inD", "type": "u32", "value": "inputDepth" }, { "name": "inH", "type": "u32", "value": "inputHeight" }, { "name": "inW", "type": "u32", "value": "inputWidth" }, { "name": "outChannels", "type": "u32", "value": "convOutChannels" }, { "name": "weightInChannels", "type": "u32", "value": "dim(shapes.w, 1)" }, { "name": "inChannelsPerGroup", "type": "u32", "value": "convInChannels / attrs.group" }, { "name": "outChannelsPerGroup", "type": "u32", "value": "convOutChannels / attrs.group" }, { "name": "kernelD", "type": "u32", "value": "kernelDepth" }, { "name": "kernelH", "type": "u32", "value": "kernelHeight" }, { "name": "kernelW", "type": "u32", "value": "kernelWidth" }, { "name": "outD", "type": "u32", "value": "outputDepth" }, { "name": "outH", "type": "u32", "value": "outputHeight" }, { "name": "outW", "type": "u32", "value": "outputWidth" }, { "name": "strideD", "type": "u32", "value": "strideD" }, { "name": "strideH", "type": "u32", "value": "strideH" }, { "name": "strideW", "type": "u32", "value": "strideW" }, { "name": "dilationD", "type": "u32", "value": "dilationD" }, { "name": "dilationH", "type": "u32", "value": "dilationH" }, { "name": "dilationW", "type": "u32", "value": "dilationW" }, { "name": "padD", "type": "i32", "value": "effectivePadFront" }, { "name": "padH", "type": "i32", "value": "effectivePadTop" }, { "name": "padW", "type": "i32", "value": "effectivePadLeft" }, { "name": "count", "type": "u32", "value": "numel(shapes.y)" } ] }, "requant_params": { "name": "params", "struct": [ { "name": "count", "type": "u32", "value": "numel(shapes.y)" }, { "name": "outChannels", "type": "u32", "value": "convOutChannels" }, { "name": "outImageSize", "type": "u32", "value": "outImageSize" } ] }, "requant_params_count": { "name": "params", "struct": [{ "name": "count", "type": "u32", "value": "numel(shapes.y)" }] }, "x4_params": { "name": "params", "struct": [ { "name": "batch", "type": "u32", "value": "dim(shapes.x, 0)" }, { "name": "inChannels", "type": "u32", "value": "convInChannels" }, { "name": "inH", "type": "u32", "value": "inputHeight" }, { "name": "inW", "type": "u32", "value": "inputWidth" }, { "name": "outChannels", "type": "u32", "value": "convOutChannels" }, { "name": "weightInChannels", "type": "u32", "value": "dim(shapes.w, 1)" }, { "name": "inChannelsPerGroup", "type": "u32", "value": "convInChannels / attrs.group" }, { "name": "outChannelsPerGroup", "type": "u32", "value": "convOutChannels / attrs.group" }, { "name": "kernelH", "type": "u32", "value": "kernelHeight" }, { "name": "kernelW", "type": "u32", "value": "kernelWidth" }, { "name": "outH", "type": "u32", "value": "outputHeight" }, { "name": "outW", "type": "u32", "value": "outputWidth" }, { "name": "strideH", "type": "u32", "value": "strideH" }, { "name": "strideW", "type": "u32", "value": "strideW" }, { "name": "dilationH", "type": "u32", "value": "dilationH" }, { "name": "dilationW", "type": "u32", "value": "dilationW" }, { "name": "padH", "type": "i32", "value": "effectivePadTop" }, { "name": "padW", "type": "i32", "value": "effectivePadLeft" } ] }, "oc4_params": { "name": "params", "struct": [ { "name": "count", "type": "u32", "value": "dim(shapes.x, 0) * outputHeight * ceilDiv(outputWidth, nhwcOc4ColumnTile) * ceilDiv(convOutChannels, 4)" }, { "name": "ocBlocks", "type": "u32", "value": "ceilDiv(convOutChannels, 4)" }, { "name": "owGroups", "type": "u32", "value": "ceilDiv(outputWidth, nhwcOc4ColumnTile)" }, { "name": "inChannels", "type": "u32", "value": "convInChannels" }, { "name": "inH", "type": "u32", "value": "inputHeight" }, { "name": "inW", "type": "u32", "value": "inputWidth" }, { "name": "outChannels", "type": "u32", "value": "convOutChannels" }, { "name": "weightInChannels", "type": "u32", "value": "dim(shapes.w, 1)" }, { "name": "inChannelsPerGroup", "type": "u32", "value": "convInChannels / attrs.group" }, { "name": "outChannelsPerGroup", "type": "u32", "value": "convOutChannels / attrs.group" }, { "name": "kernelH", "type": "u32", "value": "kernelHeight" }, { "name": "kernelW", "type": "u32", "value": "kernelWidth" }, { "name": "outH", "type": "u32", "value": "outputHeight" }, { "name": "outW", "type": "u32", "value": "outputWidth" }, { "name": "strideH", "type": "u32", "value": "strideH" }, { "name": "strideW", "type": "u32", "value": "strideW" }, { "name": "dilationH", "type": "u32", "value": "dilationH" }, { "name": "dilationW", "type": "u32", "value": "dilationW" }, { "name": "padH", "type": "i32", "value": "effectivePadTop" }, { "name": "padW", "type": "i32", "value": "effectivePadLeft" } ] }, "a_weight": { "arg": "w", "name": "a", "elementType": "$aVec4" }, "a_zero_point_weight": { "arg": "w_zero_point", "name": "a_zero_point", "elementType": "$aScalar" }, "b_x": { "arg": "x", "name": "b", "elementType": "$bScalar" }, "b_zero_point_x": { "arg": "x_zero_point", "name": "b_zero_point", "elementType": "$bScalar", "length": 1 }, "b_cols_nchw": { "scratch": "cols", "name": "b", "buffer": "read-only-storage", "elementType": "$bScalar" }, "a_x": { "arg": "x", "name": "a", "elementType": "$aVec4" }, "a_cols_nhwc": { "scratch": "cols", "name": "a", "buffer": "read-only-storage", "elementType": "$aVec4" }, "a_zero_point_x": { "arg": "x_zero_point", "name": "a_zero_point", "elementType": "$aScalar", "length": 1 }, "b_weight_k_major": { "scratch": "w_k_major", "name": "b", "buffer": "read-only-storage", "elementType": "$bScalar" }, "b_zero_point_weight": { "arg": "w_zero_point", "name": "b_zero_point", "elementType": "$bScalar" }, "a_scale_x": { "arg": "x_scale", "name": "a_scale", "elementType": "f32", "length": 1 }, "b_scale_w": { "arg": "w_scale", "name": "b_scale", "elementType": "f32" }, "bias": { "arg": "B", "name": "bias", "elementType": "i32" }, "w_k_major_out": { "scratch": "w_k_major", "name": "w_k_major", "elementType": "$wScalar" }, "k_major_params": { "name": "params", "struct": [ { "name": "count", "type": "u32", "value": "convKPadded * convOutChannels" }, { "name": "outChannels", "type": "u32", "value": "convOutChannels" }, { "name": "weightInChannels", "type": "u32", "value": "dim(shapes.w, 1)" }, { "name": "kernelH", "type": "u32", "value": "kernelHeight" }, { "name": "kernelW", "type": "u32", "value": "kernelWidth" }, { "name": "kRows", "type": "u32", "value": "convK" } ] }, "k_major_params_exact": { "name": "params", "struct": [ { "name": "count", "type": "u32", "value": "convK * convOutChannels" }, { "name": "outChannels", "type": "u32", "value": "convOutChannels" }, { "name": "weightInChannels", "type": "u32", "value": "dim(shapes.w, 1)" }, { "name": "kernelH", "type": "u32", "value": "kernelHeight" }, { "name": "kernelW", "type": "u32", "value": "kernelWidth" }, { "name": "kRows", "type": "u32", "value": "convK" } ] }, "x_im2col": { "name": "x", "elementType": "$xVec4" }, "cols_out_vec4": { "scratch": "cols", "name": "cols", "elementType": "$colsVec4" }, "im2col_nhwc_params": { "name": "params", "struct": [ { "name": "count", "type": "u32", "value": "ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)" }, { "name": "totalWords", "type": "u32", "value": "dim(shapes.x, 0) * outSpatial * (convKPadded / 4)" }, { "name": "wordsPerRow", "type": "u32", "value": "convKPadded / 4" }, { "name": "kRows", "type": "u32", "value": "convK" }, { "name": "outCount", "type": "u32", "value": "outSpatial" }, { "name": "inChannels", "type": "u32", "value": "convInChannels" }, { "name": "inH", "type": "u32", "value": "inputHeight" }, { "name": "inW", "type": "u32", "value": "inputWidth" }, { "name": "outW", "type": "u32", "value": "outputWidth" } ] }, "im2col_nchw_params": { "name": "params", "struct": [ { "name": "outCount", "type": "u32", "value": "outSpatial" }, { "name": "inChannels", "type": "u32", "value": "convInChannels" }, { "name": "inH", "type": "u32", "value": "inputHeight" }, { "name": "inW", "type": "u32", "value": "inputWidth" }, { "name": "kRows", "type": "u32", "value": "convK" }, { "name": "outW", "type": "u32", "value": "outputWidth" } ] }, "gemm_params_nchw": { "name": "params", "struct": [ { "name": "M", "type": "u32", "value": "convOutChannels" }, { "name": "N", "type": "u32", "value": "outSpatial" }, { "name": "K", "type": "u32", "value": "convK" }, { "name": "aBatchStride4", "type": "u32", "value": 0 }, { "name": "bBatchStride", "type": "u32", "value": "convK * outSpatial" }, { "name": "yBatchStride", "type": "u32", "value": "convOutChannels * outSpatial" } ] }, "gemm_params_nhwc_pointwise": { "name": "params", "struct": [ { "name": "M", "type": "u32", "value": "outSpatial" }, { "name": "N", "type": "u32", "value": "convOutChannels" }, { "name": "K", "type": "u32", "value": "convK" }, { "name": "aBatchStride4", "type": "u32", "value": "(outSpatial * convK / 4) if convK % 4 == 0 else (outSpatial * convK)" }, { "name": "bBatchStride", "type": "u32", "value": 0 }, { "name": "yBatchStride", "type": "u32", "value": "convOutChannels * outSpatial" } ] }, "gemm_params_nhwc_im2col": { "name": "params", "struct": [ { "name": "M", "type": "u32", "value": "outSpatial" }, { "name": "N", "type": "u32", "value": "convOutChannels" }, { "name": "K", "type": "u32", "value": "convKPadded" }, { "name": "aBatchStride4", "type": "u32", "value": "outSpatial * convKPadded / 4" }, { "name": "bBatchStride", "type": "u32", "value": 0 }, { "name": "yBatchStride", "type": "u32", "value": "convOutChannels * outSpatial" } ] }, "x_b": { "name": "x", "elementType": "$bScalar" }, "x_zero_point_b": { "name": "x_zero_point", "elementType": "$bScalar", "length": 1 }, "cols_out_b": { "scratch": "cols", "name": "cols", "elementType": "$bScalar" }, "gemm_params_nchw_matrix": { "name": "params", "struct": [ { "name": "M", "type": "u32", "value": "sgmatM" }, { "name": "N", "type": "u32", "value": "outSpatial" }, { "name": "K", "type": "u32", "value": "convK" }, { "name": "bBatchStride", "type": "u32", "value": "convK * outSpatial" }, { "name": "yBatchStride", "type": "u32", "value": "convOutChannels * outSpatial" } ] }, "gemm_params_nhwc_pointwise_matrix": { "name": "params", "struct": [ { "name": "M", "type": "u32", "value": "sgmatM" }, { "name": "N", "type": "u32", "value": "convOutChannels" }, { "name": "K", "type": "u32", "value": "convK" } ] }, "gemm_params_nhwc_im2col_matrix": { "name": "params", "struct": [ { "name": "M", "type": "u32", "value": "sgmatM" }, { "name": "N", "type": "u32", "value": "convOutChannels" }, { "name": "K", "type": "u32", "value": "convKPadded" } ] }, "precast_aOut": { "scratch": "aFloat", "name": "aFloat", "elementType": "$precastScalar" }, "precast_bOut": { "scratch": "bFloat", "name": "bFloat", "elementType": "$precastScalar" }, "precast_a": { "scratch": "aFloat", "name": "a", "buffer": "read-only-storage", "elementType": "$precastScalar" }, "precast_b": { "scratch": "bFloat", "name": "b", "buffer": "read-only-storage", "elementType": "$precastScalar" }, "precast_matrix_params": { "name": "params", "struct": [ { "name": "M", "type": "u32", "value": "convOutChannels" }, { "name": "N", "type": "u32", "value": "outSpatial" }, { "name": "K", "type": "u32", "value": "convK" }, { "name": "yBatchStride", "type": "u32", "value": "convOutChannels * outSpatial" } ] }, "precast_prepare_plain": { "name": "params", "struct": [ { "name": "M", "type": "u32", "value": "sgmatM" }, { "name": "K", "type": "u32", "value": "convK" }, { "name": "paddedK", "type": "u32", "value": "preparedK" }, { "name": "count", "type": "u32", "value": "prepareTotalElements" }, { "name": "groupsX", "type": "u32", "value": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" }, { "name": "N", "type": "u32", "value": "sgmatN" }, { "name": "paddedN", "type": "u32", "value": "preparedN" }, { "name": "aCount", "type": "u32", "value": "preparedElements" } ] }, "precast_prepare_spatial": { "name": "params", "struct": [ { "name": "M", "type": "u32", "value": "sgmatM" }, { "name": "K", "type": "u32", "value": "convK" }, { "name": "paddedK", "type": "u32", "value": "preparedK" }, { "name": "count", "type": "u32", "value": "prepareTotalElements" }, { "name": "groupsX", "type": "u32", "value": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" }, { "name": "N", "type": "u32", "value": "sgmatN" }, { "name": "paddedN", "type": "u32", "value": "preparedN" }, { "name": "aCount", "type": "u32", "value": "preparedElements" }, { "name": "inChannels", "type": "u32", "value": "convInChannels" }, { "name": "inH", "type": "u32", "value": "inputHeight" }, { "name": "inW", "type": "u32", "value": "inputWidth" }, { "name": "outW", "type": "u32", "value": "outputWidth" } ] }, "precast_matrix_nhwc": { "name": "params", "struct": [ { "name": "M", "type": "u32", "value": "sgmatM" }, { "name": "N", "type": "u32", "value": "sgmatN" }, { "name": "K", "type": "u32", "value": "convK" } ] }, "precast_weight_oihw": { "arg": "w", "name": "b", "elementType": "$bScalar" }, "precast_prepare_nhwc_plain": { "name": "params", "struct": [ { "name": "M", "type": "u32", "value": "sgmatM" }, { "name": "K", "type": "u32", "value": "convK" }, { "name": "paddedK", "type": "u32", "value": "preparedK" }, { "name": "count", "type": "u32", "value": "prepareTotalElements" }, { "name": "groupsX", "type": "u32", "value": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" }, { "name": "N", "type": "u32", "value": "sgmatN" }, { "name": "paddedN", "type": "u32", "value": "preparedN" }, { "name": "aCount", "type": "u32", "value": "preparedElements" } ] }, "precast_prepare_nhwc_spatial": { "name": "params", "struct": [ { "name": "M", "type": "u32", "value": "sgmatM" }, { "name": "K", "type": "u32", "value": "convK" }, { "name": "paddedK", "type": "u32", "value": "preparedK" }, { "name": "count", "type": "u32", "value": "prepareTotalElements" }, { "name": "groupsX", "type": "u32", "value": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" }, { "name": "N", "type": "u32", "value": "sgmatN" }, { "name": "paddedN", "type": "u32", "value": "preparedN" }, { "name": "aCount", "type": "u32", "value": "preparedElements" }, { "name": "outCount", "type": "u32", "value": "outSpatial" }, { "name": "inChannels", "type": "u32", "value": "convInChannels" }, { "name": "inH", "type": "u32", "value": "inputHeight" }, { "name": "inW", "type": "u32", "value": "inputWidth" }, { "name": "outW", "type": "u32", "value": "outputWidth" } ] }, "portable_f16_a": { "name": "a", "scratch": "aFloat", "buffer": "read-only-storage", "elementType": "vec4" }, "portable_f16_b": { "name": "b", "scratch": "bFloat", "buffer": "read-only-storage", "elementType": "vec4" }, "portable_f32_a": { "name": "a", "scratch": "aFloat", "buffer": "read-only-storage", "elementType": "vec4" }, "portable_f32_b": { "name": "b", "scratch": "bFloat", "buffer": "read-only-storage", "elementType": "vec4" } }, "variants": [ { "id": "spatial_nd2_scalar", "when": ["(rank3TensorOk or rank4TensorOk)", "scalarWeightQuantOk", "not present.B", "groupChannelsOk", "outputSpatialOk"], "derive": { "hasBias": false, "wScalePerChannel": false, "wZeroPerChannel": false }, "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }], "passes": [ { "id": "accumulate", "name": "QLinearConv.Accumulate", "shader": "conv-int-accumulate-spatial.wgsl.jinja", "derive": { "spatialRankSpec": 2 }, "bindings": ["x_w", "x_zero_point_w", "w_w", "w_zero_point_w", "accum_out", "accum_params_2d"], "dispatch": { "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } }, { "id": "requantize", "name": "QLinearConv.Requantize", "shader": "qlinear-conv-requantize.wgsl.jinja", "bindings": ["accum_in", "x_scale", "w_scale", "y_scale", "y_zero_point_w", "y_w", "requant_params_count"], "dispatch": { "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } } ] }, { "id": "spatial_nd2_scalar_bias", "when": ["(rank3TensorOk or rank4TensorOk)", "scalarWeightQuantOk", "biasContractOk", "groupChannelsOk", "outputSpatialOk"], "derive": { "hasBias": true, "wScalePerChannel": false, "wZeroPerChannel": false }, "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }], "passes": [ { "id": "accumulate", "name": "QLinearConv.Accumulate", "shader": "conv-int-accumulate-spatial.wgsl.jinja", "derive": { "spatialRankSpec": 2 }, "bindings": ["x_w", "x_zero_point_w", "w_w", "w_zero_point_w", "accum_out", "accum_params_2d"], "dispatch": { "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } }, { "id": "requantize", "name": "QLinearConv.Requantize", "shader": "qlinear-conv-requantize.wgsl.jinja", "bindings": ["accum_in", "x_scale", "w_scale", "y_scale", "y_zero_point_w", "B", "y_w", "requant_params"], "dispatch": { "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } } ] }, { "id": "spatial_nd2_per_channel", "when": ["(rank3TensorOk or rank4TensorOk)", "perChannelWeightQuantOk", "not present.B", "groupChannelsOk", "outputSpatialOk"], "derive": { "hasBias": false, "wScalePerChannel": true, "wZeroPerChannel": true }, "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }], "passes": [ { "id": "accumulate", "name": "QLinearConv.Accumulate", "shader": "conv-int-accumulate-spatial.wgsl.jinja", "derive": { "spatialRankSpec": 2 }, "bindings": ["x_w", "x_zero_point_w", "w_w", "w_zero_point_w", "accum_out", "accum_params_2d"], "dispatch": { "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } }, { "id": "requantize", "name": "QLinearConv.Requantize", "shader": "qlinear-conv-requantize.wgsl.jinja", "bindings": ["accum_in", "x_scale", "w_scale", "y_scale", "y_zero_point_w", "y_w", "requant_params"], "dispatch": { "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } } ] }, { "id": "spatial_nd2_per_channel_bias", "when": ["(rank3TensorOk or rank4TensorOk)", "perChannelWeightQuantOk", "biasContractOk", "groupChannelsOk", "outputSpatialOk"], "derive": { "hasBias": true, "wScalePerChannel": true, "wZeroPerChannel": true }, "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }], "passes": [ { "id": "accumulate", "name": "QLinearConv.Accumulate", "shader": "conv-int-accumulate-spatial.wgsl.jinja", "derive": { "spatialRankSpec": 2 }, "bindings": ["x_w", "x_zero_point_w", "w_w", "w_zero_point_w", "accum_out", "accum_params_2d"], "dispatch": { "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } }, { "id": "requantize", "name": "QLinearConv.Requantize", "shader": "qlinear-conv-requantize.wgsl.jinja", "bindings": ["accum_in", "x_scale", "w_scale", "y_scale", "y_zero_point_w", "B", "y_w", "requant_params"], "dispatch": { "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } } ] }, { "id": "spatial_nd3_scalar", "when": ["rank5TensorOk", "scalarWeightQuantOk", "not present.B", "groupChannelsOk", "outputSpatialOk"], "derive": { "hasBias": false, "wScalePerChannel": false, "wZeroPerChannel": false }, "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }], "passes": [ { "id": "accumulate", "name": "QLinearConv.Accumulate3d", "shader": "conv-int-accumulate-spatial.wgsl.jinja", "derive": { "spatialRankSpec": 3 }, "bindings": ["x_w", "x_zero_point_w", "w_w", "w_zero_point_w", "accum_out", "accum_params_3d"], "dispatch": { "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } }, { "id": "requantize", "name": "QLinearConv.Requantize", "shader": "qlinear-conv-requantize.wgsl.jinja", "bindings": ["accum_in", "x_scale", "w_scale", "y_scale", "y_zero_point_w", "y_w", "requant_params_count"], "dispatch": { "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } } ] }, { "id": "spatial_nd3_scalar_bias", "when": ["rank5TensorOk", "scalarWeightQuantOk", "biasContractOk", "groupChannelsOk", "outputSpatialOk"], "derive": { "hasBias": true, "wScalePerChannel": false, "wZeroPerChannel": false }, "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }], "passes": [ { "id": "accumulate", "name": "QLinearConv.Accumulate3d", "shader": "conv-int-accumulate-spatial.wgsl.jinja", "derive": { "spatialRankSpec": 3 }, "bindings": ["x_w", "x_zero_point_w", "w_w", "w_zero_point_w", "accum_out", "accum_params_3d"], "dispatch": { "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } }, { "id": "requantize", "name": "QLinearConv.Requantize", "shader": "qlinear-conv-requantize.wgsl.jinja", "bindings": ["accum_in", "x_scale", "w_scale", "y_scale", "y_zero_point_w", "B", "y_w", "requant_params"], "dispatch": { "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } } ] }, { "id": "spatial_nd3_per_channel", "when": ["rank5TensorOk", "perChannelWeightQuantOk", "not present.B", "groupChannelsOk", "outputSpatialOk"], "derive": { "hasBias": false, "wScalePerChannel": true, "wZeroPerChannel": true }, "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }], "passes": [ { "id": "accumulate", "name": "QLinearConv.Accumulate3d", "shader": "conv-int-accumulate-spatial.wgsl.jinja", "derive": { "spatialRankSpec": 3 }, "bindings": ["x_w", "x_zero_point_w", "w_w", "w_zero_point_w", "accum_out", "accum_params_3d"], "dispatch": { "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } }, { "id": "requantize", "name": "QLinearConv.Requantize", "shader": "qlinear-conv-requantize.wgsl.jinja", "bindings": ["accum_in", "x_scale", "w_scale", "y_scale", "y_zero_point_w", "y_w", "requant_params"], "dispatch": { "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } } ] }, { "id": "spatial_nd3_per_channel_bias", "when": ["rank5TensorOk", "perChannelWeightQuantOk", "biasContractOk", "groupChannelsOk", "outputSpatialOk"], "derive": { "hasBias": true, "wScalePerChannel": true, "wZeroPerChannel": true }, "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }], "passes": [ { "id": "accumulate", "name": "QLinearConv.Accumulate3d", "shader": "conv-int-accumulate-spatial.wgsl.jinja", "derive": { "spatialRankSpec": 3 }, "bindings": ["x_w", "x_zero_point_w", "w_w", "w_zero_point_w", "accum_out", "accum_params_3d"], "dispatch": { "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } }, { "id": "requantize", "name": "QLinearConv.Requantize", "shader": "qlinear-conv-requantize.wgsl.jinja", "bindings": ["accum_in", "x_scale", "w_scale", "y_scale", "y_zero_point_w", "B", "y_w", "requant_params"], "dispatch": { "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } } ] }, { "id": "nchw2d_columns_x4", "priority": 10, "when": ["attrs.channels_last == 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "groupChannelsOk", "outputSpatialOk", "outputWidth >= 4", "convK >= 4", "numel(shapes.y) >= 32"], "derive": { "wScalePerChannel": "perChannelWeightQuantOk", "ocTile": "4 if convOutChannels >= attrs.group * 4 and convOutChannels % (attrs.group * 4) == 0 else 1" }, "passes": [ { "id": "main", "name": "QLinearConv.ColumnsX4", "shader": "qlinear-conv-nchw-x4.wgsl.jinja", "bindings": ["x_w", "x_scale", "x_zero_point_w", "w_w", "w_scale", "w_zero_point_w", "y_scale", "y_zero_point_w", "y_w", "x4_params"], "dispatch": { "x": "min(ceilDiv((dim(shapes.y, 0) * ceilDiv(convOutChannels, 4 if convOutChannels >= attrs.group * 4 and convOutChannels % (attrs.group * 4) == 0 else 1) * outputHeight * ceilDiv(outputWidth, 4)), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((dim(shapes.y, 0) * ceilDiv(convOutChannels, 4 if convOutChannels >= attrs.group * 4 and convOutChannels % (attrs.group * 4) == 0 else 1) * outputHeight * ceilDiv(outputWidth, 4)), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } } ] }, { "id": "nhwc2d_channels_x4", "priority": 10, "when": ["attrs.channels_last != 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "groupChannelsOk", "outputSpatialOk", "convOutChannels % (attrs.group * 4) == 0", "numel(shapes.y) >= 32"], "derive": { "wScalePerChannel": "perChannelWeightQuantOk", "depthwise": false, "owTile": "nhwcOc4ColumnTile", "hasBias": false }, "passes": [ { "id": "main", "name": "QLinearConv.ChannelsX4", "shader": "qlinear-conv-nhwc-oc4.wgsl.jinja", "bindings": ["x_w", "x_scale", "x_zero_point_w", "w_w", "w_scale", "w_zero_point_w", "y_scale", "y_zero_point_w", "y_w", "oc4_params"], "dispatch": { "x": "min(ceilDiv((nhwcOc4Threads), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((nhwcOc4Threads), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } } ] }, { "id": "nhwc2d_depthwise_x4", "priority": 10, "when": ["attrs.channels_last != 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "groupChannelsOk", "outputSpatialOk", "attrs.group == convInChannels", "attrs.group == convOutChannels", "convOutChannels % 4 == 0", "numel(shapes.y) >= 32"], "derive": { "wScalePerChannel": "perChannelWeightQuantOk", "depthwise": true, "owTile": "nhwcOc4ColumnTile", "hasBias": false }, "passes": [ { "id": "main", "name": "QLinearConv.ChannelsX4", "shader": "qlinear-conv-nhwc-oc4.wgsl.jinja", "bindings": ["x_w", "x_scale", "x_zero_point_w", "w_w", "w_scale", "w_zero_point_w", "y_scale", "y_zero_point_w", "y_w", "oc4_params"], "dispatch": { "x": "min(ceilDiv((nhwcOc4Threads), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((nhwcOc4Threads), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } } ] }, { "id": "nhwc2d_channels_x4_bias", "priority": 10, "when": ["attrs.channels_last != 0", "biasContractOk", "rank4TensorOk", "flexibleWeightQuantOk", "groupChannelsOk", "outputSpatialOk", "convOutChannels % (attrs.group * 4) == 0", "numel(shapes.y) >= 32", "attrs.group > 1", "device.limits.maxStorageBuffersPerShaderStage >= 10", "tunables.WORKGROUP_SIZE <= deviceWorkgroupCap", "tunables.WORKGROUP_SIZE <= device.limits.maxComputeWorkgroupSizeX", "nhwcOc4DispatchOk"], "derive": { "wScalePerChannel": "perChannelWeightQuantOk", "depthwise": false, "owTile": "nhwcOc4ColumnTile", "hasBias": true }, "passes": [ { "id": "main", "name": "QLinearConv.ChannelsX4", "shader": "qlinear-conv-nhwc-oc4.wgsl.jinja", "bindings": ["x_w", "x_scale", "x_zero_point_w", "w_w", "w_scale", "w_zero_point_w", "y_scale", "y_zero_point_w", "y_w", "oc4_params", "bias"], "dispatch": { "x": "min(ceilDiv((nhwcOc4Threads), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((nhwcOc4Threads), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } } ] }, { "id": "nhwc2d_depthwise_x4_bias", "priority": 10, "when": ["attrs.channels_last != 0", "biasContractOk", "rank4TensorOk", "flexibleWeightQuantOk", "groupChannelsOk", "outputSpatialOk", "attrs.group == convInChannels", "attrs.group == convOutChannels", "convOutChannels % 4 == 0", "numel(shapes.y) >= 32", "attrs.group > 1", "device.limits.maxStorageBuffersPerShaderStage >= 10", "tunables.WORKGROUP_SIZE <= deviceWorkgroupCap", "tunables.WORKGROUP_SIZE <= device.limits.maxComputeWorkgroupSizeX", "nhwcOc4DispatchOk"], "derive": { "wScalePerChannel": "perChannelWeightQuantOk", "depthwise": true, "owTile": "nhwcOc4ColumnTile", "hasBias": true }, "passes": [ { "id": "main", "name": "QLinearConv.ChannelsX4", "shader": "qlinear-conv-nhwc-oc4.wgsl.jinja", "bindings": ["x_w", "x_scale", "x_zero_point_w", "w_w", "w_scale", "w_zero_point_w", "y_scale", "y_zero_point_w", "y_w", "oc4_params", "bias"], "dispatch": { "x": "min(ceilDiv((nhwcOc4Threads), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((nhwcOc4Threads), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } } ] }, { "id": "dp4a_pointwise_nchw", "priority": 20, "when": ["packedFeature", "attrs.channels_last == 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "kernelHeight == 1", "kernelWidth == 1", "strideH == 1", "strideW == 1", "padTop == 0 and padBottom == 0", "padLeft == 0 and padRight == 0", "dim(shapes.w, 1) == convInChannels", "outputHeight == inputHeight", "outputWidth == inputWidth", "gemmDispatchOk"], "derive": { "aPacked": "convK % 4 == 0", "aVec4": "(\"vec4\" if aUnsigned else \"vec4\") if aPacked else aScalar", "hasBias": false, "biasPerColumn": "channelsLast", "tileM": "4 if dp4aAppleCompact else 8", "tileN": 64, "ktWords": "dp4aCompactKWords" }, "passes": [ { "id": "main", "name": "QLinearConv.PointwiseDp4a", "shader": "quant-dp4a-matmul.wgsl.jinja", "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "gemm_params_nchw"], "dispatch": { "x": "ceilDiv(outSpatial, 64)", "y": "ceilDiv(convOutChannels, 4 if dp4aAppleCompact else 8)", "z": "dim(shapes.y, 0)" } } ] }, { "id": "matrix_pointwise_nchw", "priority": 30, "when": ["attrs.channels_last == 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "kernelHeight == 1", "kernelWidth == 1", "strideH == 1", "strideW == 1", "padTop == 0 and padBottom == 0", "padLeft == 0 and padRight == 0", "dim(shapes.w, 1) == convInChannels", "outputHeight == inputHeight", "outputWidth == inputWidth", "gemmDispatchOk", "wave32Effective", "sgmatDeviceOk", "sgmatDispatchOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["not (sgmatM >= sgmatTileM and sgmatN >= sgmatTileN / 2 and convK >= sgmatTileK)", "sgmatUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix"], "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 9 } }, "passes": [ { "id": "main", "name": "QLinearConv.ExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "derive": { "aVec4": "aScalar", "matrixHasBias": false, "matrixScalePerRow": "perChannelWeightQuantOk and not channelsLast", "matrixZeroPerRow": "perChannelWeightQuantOk and not channelsLast" }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "gemm_params_nchw_matrix"], "dispatch": { "x": "ceilDiv(sgmatN, sgmatTileN)", "y": "ceilDiv(sgmatM, sgmatTileM)", "z": "dim(shapes.x, 0)" } } ] }, { "id": "dp4a_im2col_nchw", "priority": 20, "when": ["packedFeature", "attrs.channels_last == 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "outputSpatialOk", "(kernelHeight > 1 or kernelWidth > 1)", "convK >= 64", "convK <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "nchwColsStorageOk", "gemmDispatchOk", "numel(shapes.y) >= 8192"], "derive": { "aPacked": "convK % 4 == 0", "aVec4": "(\"vec4\" if aUnsigned else \"vec4\") if aPacked else aScalar", "hasBias": false, "biasPerColumn": "channelsLast", "tileM": "4 if dp4aAppleCompact else 8", "tileN": 64, "ktWords": "dp4aCompactKWords" }, "intermediates": [{ "id": "cols", "dtype": "int32", "shape": "[dim(shapes.x, 0) * convK * outSpatial]" }], "passes": [ { "id": "im2col", "name": "QLinearConv.Im2Col", "shader": "conv-int-im2col-spatial.wgsl.jinja", "derive": { "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft" }, "bindings": ["x_b", "x_zero_point_b", "cols_out_b", "im2col_nchw_params"], "dispatch": { "x": "ceilDiv(outSpatial, tunables.WORKGROUP_SIZE)", "y": "convK", "z": "dim(shapes.x, 0)" } }, { "id": "main", "name": "QLinearConv.Im2ColDp4a", "shader": "quant-dp4a-matmul.wgsl.jinja", "bindings": ["a_weight", "b_cols_nchw", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "gemm_params_nchw"], "dispatch": { "x": "ceilDiv(outSpatial, 64)", "y": "ceilDiv(convOutChannels, 4 if dp4aAppleCompact else 8)", "z": "dim(shapes.y, 0)" } } ] }, { "id": "matrix_im2col_nchw", "priority": 30, "when": ["attrs.channels_last == 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "outputSpatialOk", "(kernelHeight > 1 or kernelWidth > 1)", "convK >= 64", "convK <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "nchwColsStorageOk", "gemmDispatchOk", "wave32Effective", "sgmatDeviceOk", "sgmatDispatchOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["not (sgmatM >= sgmatTileM and sgmatN >= sgmatTileN / 2 and convK >= sgmatTileK)", "sgmatUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix"], "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 9 } }, "intermediates": [{ "id": "cols", "dtype": "int32", "shape": "[dim(shapes.x, 0) * convK * outSpatial]" }], "passes": [ { "id": "im2col", "name": "QLinearConv.Im2Col", "shader": "conv-int-im2col-spatial.wgsl.jinja", "derive": { "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft" }, "bindings": ["x_b", "x_zero_point_b", "cols_out_b", "im2col_nchw_params"], "dispatch": { "x": "ceilDiv(outSpatial, tunables.WORKGROUP_SIZE)", "y": "convK", "z": "dim(shapes.x, 0)" } }, { "id": "main", "name": "QLinearConv.ExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "derive": { "aVec4": "aScalar", "matrixHasBias": false, "matrixScalePerRow": "perChannelWeightQuantOk and not channelsLast", "matrixZeroPerRow": "perChannelWeightQuantOk and not channelsLast" }, "bindings": ["a_weight", "b_cols_nchw", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "gemm_params_nchw_matrix"], "dispatch": { "x": "ceilDiv(sgmatN, sgmatTileN)", "y": "ceilDiv(sgmatM, sgmatTileM)", "z": "dim(shapes.x, 0)" } } ] }, { "id": "dp4a_im2col_nchw_bias", "priority": 25, "when": ["packedFeature", "attrs.channels_last == 0", "biasContractOk", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "outputSpatialOk", "convK <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "nchwColsStorageOk", "gemmDispatchOk"], "derive": { "aPacked": "convK % 4 == 0", "aVec4": "(\"vec4\" if aUnsigned else \"vec4\") if aPacked else aScalar", "hasBias": true, "biasPerColumn": "channelsLast", "tileM": "4 if dp4aAppleCompact else 8", "tileN": 64, "ktWords": "dp4aCompactKWords" }, "intermediates": [{ "id": "cols", "dtype": "int32", "shape": "[dim(shapes.x, 0) * convK * outSpatial]" }], "passes": [ { "id": "im2col", "name": "QLinearConv.Im2Col", "shader": "conv-int-im2col-spatial.wgsl.jinja", "derive": { "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft" }, "bindings": ["x_b", "x_zero_point_b", "cols_out_b", "im2col_nchw_params"], "dispatch": { "x": "ceilDiv(outSpatial, tunables.WORKGROUP_SIZE)", "y": "convK", "z": "dim(shapes.x, 0)" } }, { "id": "main", "name": "QLinearConv.Im2ColDp4aBias", "shader": "quant-dp4a-matmul.wgsl.jinja", "bindings": ["a_weight", "b_cols_nchw", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "gemm_params_nchw"], "dispatch": { "x": "ceilDiv(outSpatial, 64)", "y": "ceilDiv(convOutChannels, 4 if dp4aAppleCompact else 8)", "z": "dim(shapes.y, 0)" } } ] }, { "id": "matrix_im2col_nchw_bias", "priority": 35, "when": ["attrs.channels_last == 0", "biasContractOk", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "outputSpatialOk", "convK <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "nchwColsStorageOk", "gemmDispatchOk", "wave32Effective", "sgmatDeviceOk", "sgmatDispatchOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["not (sgmatM >= sgmatTileM and sgmatN >= sgmatTileN / 2 and convK >= sgmatTileK)", "sgmatUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix"], "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 10 } }, "intermediates": [{ "id": "cols", "dtype": "int32", "shape": "[dim(shapes.x, 0) * convK * outSpatial]" }], "passes": [ { "id": "im2col", "name": "QLinearConv.Im2Col", "shader": "conv-int-im2col-spatial.wgsl.jinja", "derive": { "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft" }, "bindings": ["x_b", "x_zero_point_b", "cols_out_b", "im2col_nchw_params"], "dispatch": { "x": "ceilDiv(outSpatial, tunables.WORKGROUP_SIZE)", "y": "convK", "z": "dim(shapes.x, 0)" } }, { "id": "main", "name": "QLinearConv.ExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "derive": { "aVec4": "aScalar", "matrixHasBias": true, "matrixScalePerRow": "perChannelWeightQuantOk and not channelsLast", "matrixZeroPerRow": "perChannelWeightQuantOk and not channelsLast" }, "bindings": ["a_weight", "b_cols_nchw", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "gemm_params_nchw_matrix"], "dispatch": { "x": "ceilDiv(sgmatN, sgmatTileN)", "y": "ceilDiv(sgmatM, sgmatTileM)", "z": "dim(shapes.x, 0)" } } ] }, { "id": "dp4a_pointwise_nhwc", "priority": 20, "when": ["packedFeature", "attrs.channels_last != 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "kernelHeight == 1", "kernelWidth == 1", "strideH == 1", "strideW == 1", "padTop == 0 and padBottom == 0", "padLeft == 0 and padRight == 0", "outputHeight == inputHeight", "outputWidth == inputWidth", "kMajorWeightStorageOk", "gemmDispatchOk", "numel(shapes.y) >= 8192"], "derive": { "aPacked": "convK % 4 == 0", "aVec4": "(\"vec4\" if aUnsigned else \"vec4\") if aPacked else aScalar", "hasBias": false, "biasPerColumn": "channelsLast", "tileM": "4 if dp4aAppleCompact else 8", "tileN": 64, "ktWords": "dp4aCompactKWords" }, "intermediates": [{ "id": "w_k_major", "dtype": "int32", "shape": "[convK * convOutChannels]" }], "passes": [ { "id": "weights", "name": "QLinearConv.WeightKMajor", "shader": "quant-weight-k-major.wgsl.jinja", "bindings": ["w_w", "w_zero_point_w", "w_k_major_out", "k_major_params_exact"], "dispatch": { "x": "min(ceilDiv((convK * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((convK * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } }, { "id": "main", "name": "QLinearConv.NhwcDp4a", "shader": "quant-dp4a-matmul.wgsl.jinja", "bindings": ["a_x", "b_weight_k_major", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "gemm_params_nhwc_pointwise"], "dispatch": { "x": "ceilDiv(convOutChannels, 64)", "y": "ceilDiv(outSpatial, 4 if dp4aAppleCompact else 8)", "z": "dim(shapes.y, 0)" } } ] }, { "id": "matrix_pointwise_nhwc", "priority": 30, "when": ["attrs.channels_last != 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "kernelHeight == 1", "kernelWidth == 1", "strideH == 1", "strideW == 1", "padTop == 0 and padBottom == 0", "padLeft == 0 and padRight == 0", "outputHeight == inputHeight", "outputWidth == inputWidth", "kMajorWeightStorageOk", "gemmDispatchOk", "wave32Effective", "sgmatDeviceOk", "sgmatDispatchOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["not (sgmatM >= sgmatTileM and sgmatN >= sgmatTileN / 2 and convK >= sgmatTileK)", "sgmatUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix"], "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 9 } }, "intermediates": [{ "id": "w_k_major", "dtype": "int32", "shape": "[convK * convOutChannels]" }], "passes": [ { "id": "weights", "name": "QLinearConv.WeightKMajor", "shader": "quant-weight-k-major.wgsl.jinja", "bindings": ["w_w", "w_zero_point_w", "w_k_major_out", "k_major_params_exact"], "dispatch": { "x": "min(ceilDiv((convK * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((convK * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } }, { "id": "main", "name": "QLinearConv.ExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "derive": { "aVec4": "aScalar", "matrixHasBias": false, "matrixScalePerRow": "perChannelWeightQuantOk and not channelsLast", "matrixZeroPerRow": "perChannelWeightQuantOk and not channelsLast" }, "bindings": ["a_x", "b_weight_k_major", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "gemm_params_nhwc_pointwise_matrix"], "dispatch": { "x": "ceilDiv(sgmatN, sgmatTileN)", "y": "ceilDiv(sgmatM, sgmatTileM)" } } ] }, { "id": "dp4a_pointwise_nhwc_bias", "priority": 25, "when": ["packedFeature", "attrs.channels_last != 0", "biasContractOk", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "kernelHeight == 1", "kernelWidth == 1", "strideH == 1", "strideW == 1", "padTop == 0 and padBottom == 0", "padLeft == 0 and padRight == 0", "outputHeight == inputHeight", "outputWidth == inputWidth", "kMajorWeightStorageOk", "gemmDispatchOk"], "derive": { "aPacked": "convK % 4 == 0", "aVec4": "(\"vec4\" if aUnsigned else \"vec4\") if aPacked else aScalar", "hasBias": true, "biasPerColumn": "channelsLast", "tileM": "4 if dp4aAppleCompact else 8", "tileN": 64, "ktWords": "dp4aCompactKWords" }, "intermediates": [{ "id": "w_k_major", "dtype": "int32", "shape": "[convK * convOutChannels]" }], "passes": [ { "id": "weights", "name": "QLinearConv.WeightKMajor", "shader": "quant-weight-k-major.wgsl.jinja", "bindings": ["w_w", "w_zero_point_w", "w_k_major_out", "k_major_params_exact"], "dispatch": { "x": "min(ceilDiv((convK * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((convK * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } }, { "id": "main", "name": "QLinearConv.NhwcDp4aBias", "shader": "quant-dp4a-matmul.wgsl.jinja", "bindings": ["a_x", "b_weight_k_major", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "gemm_params_nhwc_pointwise"], "dispatch": { "x": "ceilDiv(convOutChannels, 64)", "y": "ceilDiv(outSpatial, 4 if dp4aAppleCompact else 8)", "z": "dim(shapes.y, 0)" } } ] }, { "id": "matrix_pointwise_nhwc_bias", "priority": 35, "when": ["attrs.channels_last != 0", "biasContractOk", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "kernelHeight == 1", "kernelWidth == 1", "strideH == 1", "strideW == 1", "padTop == 0 and padBottom == 0", "padLeft == 0 and padRight == 0", "outputHeight == inputHeight", "outputWidth == inputWidth", "kMajorWeightStorageOk", "gemmDispatchOk", "wave32Effective", "sgmatDeviceOk", "sgmatDispatchOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["not (sgmatM >= sgmatTileM and sgmatN >= sgmatTileN / 2 and convK >= sgmatTileK)", "sgmatUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix"], "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 10 } }, "intermediates": [{ "id": "w_k_major", "dtype": "int32", "shape": "[convK * convOutChannels]" }], "passes": [ { "id": "weights", "name": "QLinearConv.WeightKMajor", "shader": "quant-weight-k-major.wgsl.jinja", "bindings": ["w_w", "w_zero_point_w", "w_k_major_out", "k_major_params_exact"], "dispatch": { "x": "min(ceilDiv((convK * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((convK * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } }, { "id": "main", "name": "QLinearConv.ExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "derive": { "aVec4": "aScalar", "matrixHasBias": true, "matrixScalePerRow": "perChannelWeightQuantOk and not channelsLast", "matrixZeroPerRow": "perChannelWeightQuantOk and not channelsLast" }, "bindings": ["a_x", "b_weight_k_major", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "gemm_params_nhwc_pointwise_matrix"], "dispatch": { "x": "ceilDiv(sgmatN, sgmatTileN)", "y": "ceilDiv(sgmatM, sgmatTileM)" } } ] }, { "id": "dp4a_im2col_nhwc", "priority": 20, "when": ["packedFeature", "attrs.channels_last != 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "(kernelHeight > 1 or kernelWidth > 1)", "outputSpatialOk", "nhwcColsStorageOk", "kMajorWeightStorageOk", "gemmDispatchOk", "numel(shapes.y) >= 8192"], "derive": { "aPacked": true, "aVec4": "(\"vec4\" if aUnsigned else \"vec4\") if aPacked else aScalar", "hasBias": false, "biasPerColumn": "channelsLast", "tileM": "4 if dp4aAppleCompact else 8", "tileN": 64, "ktWords": "dp4aCompactKWords" }, "intermediates": [ { "id": "w_k_major", "dtype": "int32", "shape": "[convKPadded * convOutChannels]" }, { "id": "cols", "dtype": "int32", "shape": "[dim(shapes.x, 0) * outSpatial * convKPadded]" } ], "passes": [ { "id": "weights", "name": "QLinearConv.WeightKMajor", "shader": "quant-weight-k-major.wgsl.jinja", "bindings": ["w_w", "w_zero_point_w", "w_k_major_out", "k_major_params"], "dispatch": { "x": "min(ceilDiv((convKPadded * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((convKPadded * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } }, { "id": "im2col", "name": "QLinearConv.Im2ColNhwc", "shader": "qlinear-conv-im2col-nhwc.wgsl.jinja", "derive": { "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "im2colWords": "tunables.NHWC_IM2COL_WORDS", "channelsAligned": "convInChannels % 4 == 0", "xVec4": "(\"vec4\" if tensorDtypes.x == \"uint8\" else \"vec4\") if convInChannels % 4 == 0 else (\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\")", "colsVec4": "\"vec4\" if tensorDtypes.x == \"uint8\" else \"vec4\"" }, "bindings": ["x_im2col", "x_zero_point_w", "cols_out_vec4", "im2col_nhwc_params"], "dispatch": { "x": "min(ceilDiv((ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } }, { "id": "main", "name": "QLinearConv.NhwcDp4a", "shader": "quant-dp4a-matmul.wgsl.jinja", "bindings": ["a_cols_nhwc", "b_weight_k_major", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "gemm_params_nhwc_im2col"], "dispatch": { "x": "ceilDiv(convOutChannels, 64)", "y": "ceilDiv(outSpatial, 4 if dp4aAppleCompact else 8)", "z": "dim(shapes.y, 0)" } } ] }, { "id": "matrix_im2col_nhwc", "priority": 30, "when": ["attrs.channels_last != 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "(kernelHeight > 1 or kernelWidth > 1)", "outputSpatialOk", "nhwcColsStorageOk", "kMajorWeightStorageOk", "gemmDispatchOk", "wave32Effective", "sgmatDeviceOk", "sgmatDispatchOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["not (sgmatM >= sgmatTileM and sgmatN >= sgmatTileN / 2 and convK >= sgmatTileK)", "sgmatUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix"], "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 9 } }, "intermediates": [ { "id": "w_k_major", "dtype": "int32", "shape": "[convKPadded * convOutChannels]" }, { "id": "cols", "dtype": "int32", "shape": "[dim(shapes.x, 0) * outSpatial * convKPadded]" } ], "passes": [ { "id": "weights", "name": "QLinearConv.WeightKMajor", "shader": "quant-weight-k-major.wgsl.jinja", "bindings": ["w_w", "w_zero_point_w", "w_k_major_out", "k_major_params"], "dispatch": { "x": "min(ceilDiv((convKPadded * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((convKPadded * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } }, { "id": "im2col", "name": "QLinearConv.Im2ColNhwc", "shader": "qlinear-conv-im2col-nhwc.wgsl.jinja", "derive": { "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "im2colWords": "tunables.NHWC_IM2COL_WORDS", "channelsAligned": "convInChannels % 4 == 0", "xVec4": "(\"vec4\" if tensorDtypes.x == \"uint8\" else \"vec4\") if convInChannels % 4 == 0 else (\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\")", "colsVec4": "\"vec4\" if tensorDtypes.x == \"uint8\" else \"vec4\"" }, "bindings": ["x_im2col", "x_zero_point_w", "cols_out_vec4", "im2col_nhwc_params"], "dispatch": { "x": "min(ceilDiv((ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } }, { "id": "main", "name": "QLinearConv.ExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "derive": { "aVec4": "aScalar", "matrixHasBias": false, "matrixScalePerRow": "perChannelWeightQuantOk and not channelsLast", "matrixZeroPerRow": "perChannelWeightQuantOk and not channelsLast" }, "bindings": ["a_cols_nhwc", "b_weight_k_major", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "gemm_params_nhwc_im2col_matrix"], "dispatch": { "x": "ceilDiv(sgmatN, sgmatTileN)", "y": "ceilDiv(sgmatM, sgmatTileM)" } } ] }, { "id": "dp4a_im2col_nhwc_bias", "priority": 25, "when": ["packedFeature", "attrs.channels_last != 0", "biasContractOk", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "(kernelHeight > 1 or kernelWidth > 1)", "outputSpatialOk", "nhwcColsStorageOk", "kMajorWeightStorageOk", "gemmDispatchOk"], "derive": { "aPacked": true, "aVec4": "(\"vec4\" if aUnsigned else \"vec4\") if aPacked else aScalar", "hasBias": true, "biasPerColumn": "channelsLast", "tileM": "4 if dp4aAppleCompact else 8", "tileN": 64, "ktWords": "dp4aCompactKWords" }, "intermediates": [ { "id": "w_k_major", "dtype": "int32", "shape": "[convKPadded * convOutChannels]" }, { "id": "cols", "dtype": "int32", "shape": "[dim(shapes.x, 0) * outSpatial * convKPadded]" } ], "passes": [ { "id": "weights", "name": "QLinearConv.WeightKMajor", "shader": "quant-weight-k-major.wgsl.jinja", "bindings": ["w_w", "w_zero_point_w", "w_k_major_out", "k_major_params"], "dispatch": { "x": "min(ceilDiv((convKPadded * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((convKPadded * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } }, { "id": "im2col", "name": "QLinearConv.Im2ColNhwc", "shader": "qlinear-conv-im2col-nhwc.wgsl.jinja", "derive": { "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "im2colWords": "tunables.NHWC_IM2COL_WORDS", "channelsAligned": "convInChannels % 4 == 0", "xVec4": "(\"vec4\" if tensorDtypes.x == \"uint8\" else \"vec4\") if convInChannels % 4 == 0 else (\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\")", "colsVec4": "\"vec4\" if tensorDtypes.x == \"uint8\" else \"vec4\"" }, "bindings": ["x_im2col", "x_zero_point_w", "cols_out_vec4", "im2col_nhwc_params"], "dispatch": { "x": "min(ceilDiv((ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } }, { "id": "main", "name": "QLinearConv.NhwcDp4aBias", "shader": "quant-dp4a-matmul.wgsl.jinja", "bindings": ["a_cols_nhwc", "b_weight_k_major", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "gemm_params_nhwc_im2col"], "dispatch": { "x": "ceilDiv(convOutChannels, 64)", "y": "ceilDiv(outSpatial, 4 if dp4aAppleCompact else 8)", "z": "dim(shapes.y, 0)" } } ] }, { "id": "matrix_im2col_nhwc_bias", "priority": 35, "when": ["attrs.channels_last != 0", "biasContractOk", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "(kernelHeight > 1 or kernelWidth > 1)", "outputSpatialOk", "nhwcColsStorageOk", "kMajorWeightStorageOk", "gemmDispatchOk", "wave32Effective", "sgmatDeviceOk", "sgmatDispatchOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["not (sgmatM >= sgmatTileM and sgmatN >= sgmatTileN / 2 and convK >= sgmatTileK)", "sgmatUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix"], "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 10 } }, "intermediates": [ { "id": "w_k_major", "dtype": "int32", "shape": "[convKPadded * convOutChannels]" }, { "id": "cols", "dtype": "int32", "shape": "[dim(shapes.x, 0) * outSpatial * convKPadded]" } ], "passes": [ { "id": "weights", "name": "QLinearConv.WeightKMajor", "shader": "quant-weight-k-major.wgsl.jinja", "bindings": ["w_w", "w_zero_point_w", "w_k_major_out", "k_major_params"], "dispatch": { "x": "min(ceilDiv((convKPadded * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((convKPadded * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } }, { "id": "im2col", "name": "QLinearConv.Im2ColNhwc", "shader": "qlinear-conv-im2col-nhwc.wgsl.jinja", "derive": { "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "im2colWords": "tunables.NHWC_IM2COL_WORDS", "channelsAligned": "convInChannels % 4 == 0", "xVec4": "(\"vec4\" if tensorDtypes.x == \"uint8\" else \"vec4\") if convInChannels % 4 == 0 else (\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\")", "colsVec4": "\"vec4\" if tensorDtypes.x == \"uint8\" else \"vec4\"" }, "bindings": ["x_im2col", "x_zero_point_w", "cols_out_vec4", "im2col_nhwc_params"], "dispatch": { "x": "min(ceilDiv((ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)), (tunables.WORKGROUP_SIZE)), 65535)", "y": "ceilDiv(ceilDiv((ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)), (tunables.WORKGROUP_SIZE)), 65535)", "z": 1 } }, { "id": "main", "name": "QLinearConv.ExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "derive": { "aVec4": "aScalar", "matrixHasBias": true, "matrixScalePerRow": "perChannelWeightQuantOk and not channelsLast", "matrixZeroPerRow": "perChannelWeightQuantOk and not channelsLast" }, "bindings": ["a_cols_nhwc", "b_weight_k_major", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "gemm_params_nhwc_im2col_matrix"], "dispatch": { "x": "ceilDiv(sgmatN, sgmatTileN)", "y": "ceilDiv(sgmatM, sgmatTileM)" } } ] }, { "id": "precast_f16_pointwise_nchw_tensor", "priority": 40, "when": ["precast_pointwise_nchw_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 9 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f16\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": false, "matrixScalePerRow": false, "matrixZeroPerRow": false }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": false, "sampleChannelsLast": false }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_plain"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_params"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f16_pointwise_nchw_tensor", "priority": 28, "when": ["precast_pointwise_nchw_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": ["shader-f16"] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f16\"", "scalar": "\"f16\"", "matrixHasBias": false, "matrixScalePerRow": false, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": false, "sampleChannelsLast": false }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_plain"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f16_pointwise_nchw_row", "priority": 40, "when": ["precast_pointwise_nchw_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 8 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f16\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": false, "matrixScalePerRow": true, "matrixZeroPerRow": true }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": true, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": false, "sampleChannelsLast": false }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_plain"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_params"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f16_pointwise_nchw_row", "priority": 28, "when": ["precast_pointwise_nchw_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": ["shader-f16"] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f16\"", "scalar": "\"f16\"", "matrixHasBias": false, "matrixScalePerRow": true, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": true, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": false, "sampleChannelsLast": false }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_plain"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f16_im2col_nchw_tensor", "priority": 40, "when": ["precast_im2col_nchw_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 9 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f16\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": false, "matrixScalePerRow": false, "matrixZeroPerRow": false }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": true, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": false, "sampleChannelsLast": false }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_params"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f16_im2col_nchw_tensor", "priority": 28, "when": ["precast_im2col_nchw_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": ["shader-f16"] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f16\"", "scalar": "\"f16\"", "matrixHasBias": false, "matrixScalePerRow": false, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": true, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": false, "sampleChannelsLast": false }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f16_im2col_nchw_row", "priority": 40, "when": ["precast_im2col_nchw_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 8 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f16\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": false, "matrixScalePerRow": true, "matrixZeroPerRow": true }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": true, "prepareSpatialB": true, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": false, "sampleChannelsLast": false }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_params"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f16_im2col_nchw_row", "priority": 28, "when": ["precast_im2col_nchw_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": ["shader-f16"] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f16\"", "scalar": "\"f16\"", "matrixHasBias": false, "matrixScalePerRow": true, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": true, "prepareSpatialB": true, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": false, "sampleChannelsLast": false }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f16_im2col_nchw_bias_tensor", "priority": 40, "when": ["precast_im2col_nchw_bias_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 10 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f16\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": true, "matrixScalePerRow": false, "matrixZeroPerRow": false }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": true, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": false, "sampleChannelsLast": false }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_params"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f16_im2col_nchw_bias_tensor", "priority": 28, "when": ["precast_im2col_nchw_bias_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": ["shader-f16"] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f16\"", "scalar": "\"f16\"", "matrixHasBias": true, "matrixScalePerRow": false, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": true, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": false, "sampleChannelsLast": false }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f16_im2col_nchw_bias_row", "priority": 40, "when": ["precast_im2col_nchw_bias_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 9 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f16\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": true, "matrixScalePerRow": true, "matrixZeroPerRow": true }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": true, "prepareSpatialB": true, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": false, "sampleChannelsLast": false }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_params"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f16_im2col_nchw_bias_row", "priority": 28, "when": ["precast_im2col_nchw_bias_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": ["shader-f16"] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f16\"", "scalar": "\"f16\"", "matrixHasBias": true, "matrixScalePerRow": true, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": true, "prepareSpatialB": true, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": false, "sampleChannelsLast": false }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f16_pointwise_nhwc_tensor", "priority": 40, "when": ["precast_pointwise_nhwc_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 9 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f16\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": false, "matrixScalePerRow": false, "matrixZeroPerRow": false }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_nhwc"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f16_pointwise_nhwc_tensor", "priority": 28, "when": ["precast_pointwise_nhwc_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": ["shader-f16"] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f16\"", "scalar": "\"f16\"", "matrixHasBias": false, "matrixScalePerRow": false, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f16_pointwise_nhwc_column", "priority": 40, "when": ["precast_pointwise_nhwc_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 8 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f16\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": false, "matrixScalePerRow": false, "matrixZeroPerRow": false }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "a_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_nhwc"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f16_pointwise_nhwc_column", "priority": 28, "when": ["precast_pointwise_nhwc_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": ["shader-f16"] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f16\"", "scalar": "\"f16\"", "matrixHasBias": false, "matrixScalePerRow": false, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f16_pointwise_nhwc_bias_tensor", "priority": 40, "when": ["precast_pointwise_nhwc_bias_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 10 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f16\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": true, "matrixScalePerRow": false, "matrixZeroPerRow": false }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_nhwc"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f16_pointwise_nhwc_bias_tensor", "priority": 28, "when": ["precast_pointwise_nhwc_bias_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": ["shader-f16"] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f16\"", "scalar": "\"f16\"", "matrixHasBias": true, "matrixScalePerRow": false, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f16_pointwise_nhwc_bias_column", "priority": 40, "when": ["precast_pointwise_nhwc_bias_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 9 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f16\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": true, "matrixScalePerRow": false, "matrixZeroPerRow": false }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "a_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_nhwc"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f16_pointwise_nhwc_bias_column", "priority": 28, "when": ["precast_pointwise_nhwc_bias_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": ["shader-f16"] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f16\"", "scalar": "\"f16\"", "matrixHasBias": true, "matrixScalePerRow": false, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f16_im2col_nhwc_tensor", "priority": 40, "when": ["precast_im2col_nhwc_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 9 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f16\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": false, "matrixScalePerRow": false, "matrixZeroPerRow": false }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": true, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_nhwc"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f16_im2col_nhwc_tensor", "priority": 28, "when": ["precast_im2col_nhwc_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": ["shader-f16"] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f16\"", "scalar": "\"f16\"", "matrixHasBias": false, "matrixScalePerRow": false, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": true, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f16_im2col_nhwc_column", "priority": 40, "when": ["precast_im2col_nhwc_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 8 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f16\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": false, "matrixScalePerRow": false, "matrixZeroPerRow": false }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": true, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "a_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_nhwc"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f16_im2col_nhwc_column", "priority": 28, "when": ["precast_im2col_nhwc_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": ["shader-f16"] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f16\"", "scalar": "\"f16\"", "matrixHasBias": false, "matrixScalePerRow": false, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": true, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f16_im2col_nhwc_bias_tensor", "priority": 40, "when": ["precast_im2col_nhwc_bias_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 10 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f16\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": true, "matrixScalePerRow": false, "matrixZeroPerRow": false }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": true, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_nhwc"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f16_im2col_nhwc_bias_tensor", "priority": 28, "when": ["precast_im2col_nhwc_bias_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": ["shader-f16"] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f16\"", "scalar": "\"f16\"", "matrixHasBias": true, "matrixScalePerRow": false, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": true, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f16_im2col_nhwc_bias_column", "priority": 40, "when": ["precast_im2col_nhwc_bias_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 9 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f16\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": true, "matrixScalePerRow": false, "matrixZeroPerRow": false }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": true, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "a_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_nhwc"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f16_im2col_nhwc_bias_column", "priority": 28, "when": ["precast_im2col_nhwc_bias_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": ["shader-f16"] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f16\"", "scalar": "\"f16\"", "matrixHasBias": true, "matrixScalePerRow": false, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": true, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f32_pointwise_nchw_tensor", "priority": 40, "when": ["precast_pointwise_nchw_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix"], "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 9 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f32\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": false, "matrixScalePerRow": false, "matrixZeroPerRow": false }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": false, "sampleChannelsLast": false }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_plain"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_params"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f32_pointwise_nchw_tensor", "priority": 28, "when": ["precast_pointwise_nchw_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": [] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f32\"", "scalar": "\"f32\"", "matrixHasBias": false, "matrixScalePerRow": false, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": false, "sampleChannelsLast": false }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_plain"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f32_pointwise_nchw_row", "priority": 40, "when": ["precast_pointwise_nchw_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix"], "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 8 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f32\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": false, "matrixScalePerRow": true, "matrixZeroPerRow": true }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": true, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": false, "sampleChannelsLast": false }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_plain"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_params"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f32_pointwise_nchw_row", "priority": 28, "when": ["precast_pointwise_nchw_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": [] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f32\"", "scalar": "\"f32\"", "matrixHasBias": false, "matrixScalePerRow": true, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": true, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": false, "sampleChannelsLast": false }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_plain"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f32_im2col_nchw_tensor", "priority": 40, "when": ["precast_im2col_nchw_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix"], "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 9 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f32\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": false, "matrixScalePerRow": false, "matrixZeroPerRow": false }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": true, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": false, "sampleChannelsLast": false }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_params"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f32_im2col_nchw_tensor", "priority": 28, "when": ["precast_im2col_nchw_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": [] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f32\"", "scalar": "\"f32\"", "matrixHasBias": false, "matrixScalePerRow": false, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": true, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": false, "sampleChannelsLast": false }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f32_im2col_nchw_row", "priority": 40, "when": ["precast_im2col_nchw_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix"], "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 8 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f32\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": false, "matrixScalePerRow": true, "matrixZeroPerRow": true }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": true, "prepareSpatialB": true, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": false, "sampleChannelsLast": false }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_params"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f32_im2col_nchw_row", "priority": 28, "when": ["precast_im2col_nchw_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": [] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f32\"", "scalar": "\"f32\"", "matrixHasBias": false, "matrixScalePerRow": true, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": true, "prepareSpatialB": true, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": false, "sampleChannelsLast": false }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f32_im2col_nchw_bias_tensor", "priority": 40, "when": ["precast_im2col_nchw_bias_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix"], "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 10 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f32\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": true, "matrixScalePerRow": false, "matrixZeroPerRow": false }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": true, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": false, "sampleChannelsLast": false }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_params"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f32_im2col_nchw_bias_tensor", "priority": 28, "when": ["precast_im2col_nchw_bias_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": [] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f32\"", "scalar": "\"f32\"", "matrixHasBias": true, "matrixScalePerRow": false, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": true, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": false, "sampleChannelsLast": false }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f32_im2col_nchw_bias_row", "priority": 40, "when": ["precast_im2col_nchw_bias_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix"], "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 9 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f32\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": true, "matrixScalePerRow": true, "matrixZeroPerRow": true }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": true, "prepareSpatialB": true, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": false, "sampleChannelsLast": false }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_params"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f32_im2col_nchw_bias_row", "priority": 28, "when": ["precast_im2col_nchw_bias_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": [] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f32\"", "scalar": "\"f32\"", "matrixHasBias": true, "matrixScalePerRow": true, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": true, "prepareSpatialB": true, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": false, "sampleChannelsLast": false }, "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f32_pointwise_nhwc_tensor", "priority": 40, "when": ["precast_pointwise_nhwc_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix"], "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 9 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f32\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": false, "matrixScalePerRow": false, "matrixZeroPerRow": false }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_nhwc"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f32_pointwise_nhwc_tensor", "priority": 28, "when": ["precast_pointwise_nhwc_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": [] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f32\"", "scalar": "\"f32\"", "matrixHasBias": false, "matrixScalePerRow": false, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f32_pointwise_nhwc_column", "priority": 40, "when": ["precast_pointwise_nhwc_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix"], "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 8 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f32\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": false, "matrixScalePerRow": false, "matrixZeroPerRow": false }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "a_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_nhwc"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f32_pointwise_nhwc_column", "priority": 28, "when": ["precast_pointwise_nhwc_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": [] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f32\"", "scalar": "\"f32\"", "matrixHasBias": false, "matrixScalePerRow": false, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f32_pointwise_nhwc_bias_tensor", "priority": 40, "when": ["precast_pointwise_nhwc_bias_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix"], "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 10 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f32\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": true, "matrixScalePerRow": false, "matrixZeroPerRow": false }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_nhwc"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f32_pointwise_nhwc_bias_tensor", "priority": 28, "when": ["precast_pointwise_nhwc_bias_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": [] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f32\"", "scalar": "\"f32\"", "matrixHasBias": true, "matrixScalePerRow": false, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f32_pointwise_nhwc_bias_column", "priority": 40, "when": ["precast_pointwise_nhwc_bias_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix"], "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 9 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f32\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": true, "matrixScalePerRow": false, "matrixZeroPerRow": false }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "a_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_nhwc"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f32_pointwise_nhwc_bias_column", "priority": 28, "when": ["precast_pointwise_nhwc_bias_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": [] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f32\"", "scalar": "\"f32\"", "matrixHasBias": true, "matrixScalePerRow": false, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": false, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f32_im2col_nhwc_tensor", "priority": 40, "when": ["precast_im2col_nhwc_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix"], "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 9 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f32\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": false, "matrixScalePerRow": false, "matrixZeroPerRow": false }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": true, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_nhwc"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f32_im2col_nhwc_tensor", "priority": 28, "when": ["precast_im2col_nhwc_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": [] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f32\"", "scalar": "\"f32\"", "matrixHasBias": false, "matrixScalePerRow": false, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": true, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f32_im2col_nhwc_column", "priority": 40, "when": ["precast_im2col_nhwc_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix"], "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 8 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f32\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": false, "matrixScalePerRow": false, "matrixZeroPerRow": false }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": true, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "a_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_nhwc"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f32_im2col_nhwc_column", "priority": 28, "when": ["precast_im2col_nhwc_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": [] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f32\"", "scalar": "\"f32\"", "matrixHasBias": false, "matrixScalePerRow": false, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": true, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f32_im2col_nhwc_bias_tensor", "priority": 40, "when": ["precast_im2col_nhwc_bias_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix"], "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 10 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f32\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": true, "matrixScalePerRow": false, "matrixZeroPerRow": false }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": true, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_nhwc"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f32_im2col_nhwc_bias_tensor", "priority": 28, "when": ["precast_im2col_nhwc_bias_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": [] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f32\"", "scalar": "\"f32\"", "matrixHasBias": true, "matrixScalePerRow": false, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": true, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "precast_f32_im2col_nhwc_bias_column", "priority": 40, "when": ["precast_im2col_nhwc_bias_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], "requires": { "features": ["subgroups", "chromium-experimental-subgroup-matrix"], "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], "limits": { "maxStorageBuffersPerShaderStage": 9 } }, "derive": { "aVec4": "aScalar", "precastOperands": true, "precastScalar": "\"f32\"", "sgmatTileM": "precastTileM", "sgmatTileN": "precastTileN", "sgmatTileK": "precastTileK", "sgmatRowGroups": "precastSubgroups", "sgmatSubgroups": "precastSubgroups", "sgmatWorkgroup": "precastWorkgroup", "matrixHasBias": true, "matrixScalePerRow": false, "matrixZeroPerRow": false }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": true, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactMatrix", "shader": "quant-exact-matrix.wgsl.jinja", "bindings": ["precast_a", "precast_b", "a_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_nhwc"], "dispatch": { "x": "ceilDiv(sgmatN, precastTileN)", "y": "ceilDiv(sgmatM, precastTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] }, { "id": "portable_f32_im2col_nhwc_bias_column", "priority": 28, "when": ["precast_im2col_nhwc_bias_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], "requires": { "features": [] }, "derive": { "aVec4": "aScalar", "precastScalar": "\"f32\"", "scalar": "\"f32\"", "matrixHasBias": true, "matrixScalePerRow": false, "M": "sgmatM", "N": "sgmatN", "K": "preparedK", "kTile": "portableTileK", "workgroupCols": "portableWorkgroupCols", "workgroupRows": "portableWorkgroupRows", "blockRows": "portableTileM", "blockCols": "portableTileN", "integerPartialK": "portablePartialK", "hasBias": false, "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" }, "intermediates": [ { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } ], "passes": [ { "id": "prepare", "name": "QLinearConv.PrepareExact", "shader": "quant-exact-prepare.wgsl.jinja", "derive": { "prepareZeroPerRow": false, "prepareSpatialB": false, "spatialRankSpec": 2, "kernelHSpec": "kernelHeight", "kernelWSpec": "kernelWidth", "strideHSpec": "strideH", "strideWSpec": "strideW", "dilationHSpec": "dilationH", "dilationWSpec": "dilationW", "padTopSpec": "effectivePadTop", "padLeftSpec": "effectivePadLeft", "prepareSpatialA": true, "prepareTransposeB": true, "sampleChannelsLast": true }, "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], "dispatch": { "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" } }, { "id": "main", "name": "QLinearConv.PreparedExactPortable", "shader": "quant-exact-portable.wgsl.jinja", "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], "dispatch": { "x": "ceilDiv(sgmatN, portableTileN)", "y": "ceilDiv(sgmatM, portableTileM)", "z": "1 if channelsLast else dim(shapes.x, 0)" } } ] } ] }