| { |
| "domain": "com.microsoft", |
| "name": "QLinearConv", |
| "sinceVersion": 1, |
| "inputs": { |
| "x": { "dtype": "TX" }, |
| "x_scale": { "dtype": "TF" }, |
| "x_zero_point": { "dtype": "TX" }, |
| "w": { "dtype": "TW" }, |
| "w_scale": { "dtype": "TF" }, |
| "w_zero_point": { "dtype": "TW" }, |
| "y_scale": { "dtype": "TF" }, |
| "y_zero_point": { "dtype": "TY" }, |
| "B": { "dtype": "int32", "rank": 1, "optional": true } |
| }, |
| "outputs": { |
| "y": { |
| "dtype": "TY", |
| "rank": "ranks.x", |
| "shape": "([dim(shapes.x, 0), expectedOutputWidth, dim(shapes.w, 0)] if ranks.x == 3 else ([dim(shapes.x, 0), expectedOutputHeight, expectedOutputWidth, dim(shapes.w, 0)] if ranks.x == 4 else [dim(shapes.x, 0), expectedOutputDepth, expectedOutputHeight, expectedOutputWidth, dim(shapes.w, 0)])) if channelsLast else ([dim(shapes.x, 0), dim(shapes.w, 0), expectedOutputWidth] if ranks.x == 3 else ([dim(shapes.x, 0), dim(shapes.w, 0), expectedOutputHeight, expectedOutputWidth] if ranks.x == 4 else [dim(shapes.x, 0), dim(shapes.w, 0), expectedOutputDepth, expectedOutputHeight, expectedOutputWidth]))" |
| } |
| }, |
| "attributes": { |
| "auto_pad": { "default": "NOTSET" }, |
| "channels_last": { "default": 0 }, |
| "group": { "default": 1 }, |
| "dilations": {}, |
| "kernel_shape": {}, |
| "pads": {}, |
| "strides": {} |
| }, |
| "attributeConstraints": { "auto_pad": { "values": ["NOTSET", "SAME_UPPER", "SAME_LOWER", "VALID"] } }, |
| "typeConstraints": { "TX": ["uint8", "int8"], "TW": ["uint8", "int8"], "TY": ["uint8", "int8"], "TF": ["float32"] }, |
| "tunables": { |
| "WORKGROUP_SIZE": { "default": 256 }, |
| "NHWC_OW_TILE": { "default": 4 }, |
| "NHWC_IM2COL_WORDS": { "default": 2 }, |
| "EXACT_MATRIX_MIN_INVOCATIONS": { "default": 10240 }, |
| "NHWC_BIAS_TARGET_WORKGROUPS": { "default": 64 } |
| }, |
| "derive": { |
| "channelsLast": "attrs.channels_last != 0", |
| "convInChannels": "dim(shapes.x, ranks.x - 1) if channelsLast else dim(shapes.x, 1)", |
| "convOutChannels": "dim(shapes.w, 0)", |
| "inputDepth": "(dim(shapes.x, 1) if channelsLast else dim(shapes.x, 2)) if ranks.x == 5 else 1", |
| "inputHeight": "((dim(shapes.x, 2) if channelsLast else dim(shapes.x, 3)) if ranks.x == 5 else ((dim(shapes.x, 1) if channelsLast else dim(shapes.x, 2)) if ranks.x == 4 else 1))", |
| "inputWidth": "(dim(shapes.x, ranks.x - 2) if channelsLast else dim(shapes.x, ranks.x - 1)) if ranks.x >= 3 else 1", |
| "outputDepth": "(dim(shapes.y, 1) if channelsLast else dim(shapes.y, 2)) if ranks.y == 5 else 1", |
| "outputHeight": "((dim(shapes.y, 2) if channelsLast else dim(shapes.y, 3)) if ranks.y == 5 else ((dim(shapes.y, 1) if channelsLast else dim(shapes.y, 2)) if ranks.y == 4 else 1))", |
| "outputWidth": "(dim(shapes.y, ranks.y - 2) if channelsLast else dim(shapes.y, ranks.y - 1)) if ranks.y >= 3 else 1", |
| "kernelDepth": "dim(shapes.w, 2) if ranks.w == 5 else 1", |
| "kernelHeight": "dim(shapes.w, 3) if ranks.w == 5 else (dim(shapes.w, 2) if ranks.w == 4 else 1)", |
| "kernelWidth": "dim(shapes.w, ranks.w - 1) if ranks.w >= 3 else 1", |
| "spatialRank": "ranks.w - 2", |
| "outSpatial": "outputDepth * outputHeight * outputWidth", |
| "convK": "dim(shapes.w, 1) * kernelDepth * kernelHeight * kernelWidth", |
| "dp4aCandidateKWords": 128, |
| "dp4aAppleCompact": "device.adapterInfo.vendor == \"apple\" and device.features.has(\"subgroups\") and not channelsLast and not present.B and attrs.group == 1 and (kernelHeight > 1 or kernelWidth > 1) and convOutChannels >= outSpatial and convK >= 4 * dp4aCandidateKWords and ceilDiv(convOutChannels, 4) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(outSpatial, 4) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", |
| "dp4aCompactKWords": "dp4aCandidateKWords if dp4aAppleCompact else 32", |
| "sgmatM": "dim(shapes.x, 0) * outSpatial if channelsLast else convOutChannels", |
| "sgmatN": "convOutChannels if channelsLast else outSpatial", |
| "sgmatTileRows": "sgmatM", |
| "precastRows": "sgmatM", |
| "precastContraction": "convK", |
| "precastCols": "sgmatN", |
| "precastBatches": "1 if channelsLast else dim(shapes.x, 0)", |
| "portableWorkgroupCols": 8, |
| "kernelShapeLengthOk": "not has(attrs, \"kernel_shape\") or (attrs.kernel_shape | length) == spatialRank", |
| "stridesLengthOk": "not has(attrs, \"strides\") or (attrs.strides | length) == spatialRank", |
| "dilationsLengthOk": "not has(attrs, \"dilations\") or (attrs.dilations | length) == spatialRank", |
| "padsLengthOk": "not has(attrs, \"pads\") or (attrs.pads | length) == 2 * spatialRank", |
| "kernelD": "attrs.kernel_shape[0] if kernelShapeLengthOk and has(attrs, \"kernel_shape\") and spatialRank == 3 else 1", |
| "kernelH": "attrs.kernel_shape[spatialRank - 2] if kernelShapeLengthOk and has(attrs, \"kernel_shape\") and spatialRank >= 2 else 1", |
| "kernelW": "attrs.kernel_shape[spatialRank - 1] if kernelShapeLengthOk and has(attrs, \"kernel_shape\") and spatialRank >= 1 else 1", |
| "strideD": "attrs.strides[0] if stridesLengthOk and has(attrs, \"strides\") and spatialRank == 3 else 1", |
| "strideH": "attrs.strides[spatialRank - 2] if stridesLengthOk and has(attrs, \"strides\") and spatialRank >= 2 else 1", |
| "strideW": "attrs.strides[spatialRank - 1] if stridesLengthOk and has(attrs, \"strides\") and spatialRank >= 1 else 1", |
| "dilationD": "attrs.dilations[0] if dilationsLengthOk and has(attrs, \"dilations\") and spatialRank == 3 else 1", |
| "dilationH": "attrs.dilations[spatialRank - 2] if dilationsLengthOk and has(attrs, \"dilations\") and spatialRank >= 2 else 1", |
| "dilationW": "attrs.dilations[spatialRank - 1] if dilationsLengthOk and has(attrs, \"dilations\") and spatialRank >= 1 else 1", |
| "padFront": "attrs.pads[0] if padsLengthOk and has(attrs, \"pads\") and spatialRank == 3 else 0", |
| "padTop": "attrs.pads[spatialRank - 2] if padsLengthOk and has(attrs, \"pads\") and spatialRank >= 2 else 0", |
| "padLeft": "attrs.pads[spatialRank - 1] if padsLengthOk and has(attrs, \"pads\") and spatialRank >= 1 else 0", |
| "padBack": "attrs.pads[spatialRank] if padsLengthOk and has(attrs, \"pads\") and spatialRank == 3 else 0", |
| "padBottom": "attrs.pads[2 * spatialRank - 2] if padsLengthOk and has(attrs, \"pads\") and spatialRank >= 2 else 0", |
| "padRight": "attrs.pads[2 * spatialRank - 1] if padsLengthOk and has(attrs, \"pads\") and spatialRank >= 1 else 0", |
| "autoPadSame": "attrs.auto_pad == \"SAME_UPPER\" or attrs.auto_pad == \"SAME_LOWER\"", |
| "autoPadValid": "attrs.auto_pad == \"VALID\"", |
| "samePadDepth": "max(0, (outputDepth - 1) * strideD + (kernelDepth - 1) * dilationD + 1 - inputDepth)", |
| "samePadHeight": "max(0, (outputHeight - 1) * strideH + (kernelHeight - 1) * dilationH + 1 - inputHeight)", |
| "samePadWidth": "max(0, (outputWidth - 1) * strideW + (kernelWidth - 1) * dilationW + 1 - inputWidth)", |
| "samePadFront": "floor(samePadDepth / 2) if attrs.auto_pad == \"SAME_UPPER\" else samePadDepth - floor(samePadDepth / 2)", |
| "samePadTop": "floor(samePadHeight / 2) if attrs.auto_pad == \"SAME_UPPER\" else samePadHeight - floor(samePadHeight / 2)", |
| "samePadLeft": "floor(samePadWidth / 2) if attrs.auto_pad == \"SAME_UPPER\" else samePadWidth - floor(samePadWidth / 2)", |
| "effectivePadFront": "samePadFront if autoPadSame else (0 if autoPadValid else padFront)", |
| "effectivePadTop": "samePadTop if autoPadSame else (0 if autoPadValid else padTop)", |
| "effectivePadLeft": "samePadLeft if autoPadSame else (0 if autoPadValid else padLeft)", |
| "expectedOutputDepth": "ceil(inputDepth / strideD) if autoPadSame else floor((inputDepth + (0 if autoPadValid else padFront + padBack) - ((kernelDepth - 1) * dilationD + 1)) / strideD) + 1", |
| "expectedOutputHeight": "ceil(inputHeight / strideH) if autoPadSame else floor((inputHeight + (0 if autoPadValid else padTop + padBottom) - ((kernelHeight - 1) * dilationH + 1)) / strideH) + 1", |
| "expectedOutputWidth": "ceil(inputWidth / strideW) if autoPadSame else floor((inputWidth + (0 if autoPadValid else padLeft + padRight) - ((kernelWidth - 1) * dilationW + 1)) / strideW) + 1", |
| "wave32Adapter": "has(device.adapterInfo, \"subgroupMinSize\") and has(device.adapterInfo, \"subgroupMaxSize\") and device.adapterInfo.subgroupMinSize == 32 and device.adapterInfo.subgroupMaxSize == 32", |
| "canPinSubgroupSize32": "device.features.has(\"subgroups\") and device.features.has(\"subgroup-size-control\") and has(device.adapterInfo, \"subgroupMinSize\") and has(device.adapterInfo, \"subgroupMaxSize\") and device.adapterInfo.subgroupMinSize <= 32 and device.adapterInfo.subgroupMaxSize >= 32", |
| "pinSubgroupSize32": "canPinSubgroupSize32 and not wave32Adapter", |
| "wave32Effective": "wave32Adapter or pinSubgroupSize32", |
| "deviceWorkgroupCap": "min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)", |
| "sgmatTileN": "64", |
| "sgmatTileK": "32", |
| "sgmatColumnGroups": "2", |
| "sgmatStorageRowGroups": "floor((device.limits.maxComputeWorkgroupStorageSize - sgmatTileN * sgmatTileK * 4) / (16 * sgmatTileK * 4))", |
| "sgmatRowGroups": "max(1, min(4, floor(deviceWorkgroupCap / (32 * sgmatColumnGroups)), sgmatStorageRowGroups, max(floor(sgmatStorageRowGroups / 2), floor(sgmatTileRows / 16))))", |
| "sgmatTileM": "16 * sgmatRowGroups", |
| "sgmatSubgroups": "sgmatRowGroups * sgmatColumnGroups", |
| "sgmatWorkgroup": "32 * sgmatSubgroups", |
| "sgmatStagingBytes": "(sgmatTileM + sgmatTileN) * sgmatTileK * 4", |
| "sgmatScratchBytes": "sgmatSubgroups * 4 * 64 * 4", |
| "sgmatReuseStaging": "sgmatStagingBytes + sgmatScratchBytes > device.limits.maxComputeWorkgroupStorageSize", |
| "sgmatSharedBytes": "sgmatStagingBytes + (0 if sgmatReuseStaging else sgmatScratchBytes)", |
| "sgmatDeviceOk": "sgmatWorkgroup <= deviceWorkgroupCap and sgmatSharedBytes <= device.limits.maxComputeWorkgroupStorageSize", |
| "precastSubgroups": "max(1, min(4, ceilDiv(precastRows, 16), floor(deviceWorkgroupCap / 32), floor(device.limits.maxComputeWorkgroupStorageSize / (8 * 64 * 4))))", |
| "precastTileM": "16 * precastSubgroups", |
| "precastTileN": "4 * 8", |
| "precastTileK": "4 * 8", |
| "precastWorkgroup": "32 * precastSubgroups", |
| "preparedK": "ceilDiv(precastContraction, precastTileK) * precastTileK", |
| "preparedN": "ceilDiv(precastCols, precastTileN) * precastTileN", |
| "preparedElements": "ceilDiv(precastRows, precastTileM) * precastTileM * preparedK", |
| "preparedBElements": "precastBatches * preparedK * preparedN", |
| "prepareWorkgroup": "min(256, deviceWorkgroupCap)", |
| "prepareTotalElements": "preparedElements + preparedBElements", |
| "prepareGroups": "ceilDiv(prepareTotalElements, 4 * prepareWorkgroup)", |
| "prepareDispatchElements": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535)) * min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535)) * prepareWorkgroup * 4", |
| "precastResourcesOk": "deviceWorkgroupCap >= precastWorkgroup and precastSubgroups * 8 * 64 * 4 <= device.limits.maxComputeWorkgroupStorageSize", |
| "precastDispatchOk": "prepareTotalElements <= 4294967295 and prepareDispatchElements <= 4294967296 and ceilDiv(precastCols, precastTileN) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(precastRows, precastTileM) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and prepareGroups <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) * min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", |
| "precastF32BuffersOk": "preparedElements * 4 <= device.limits.maxStorageBufferBindingSize and preparedElements * 4 <= device.limits.maxBufferSize and preparedBElements * 4 <= device.limits.maxStorageBufferBindingSize and preparedBElements * 4 <= device.limits.maxBufferSize", |
| "precastF16BuffersOk": "preparedElements * 2 <= device.limits.maxStorageBufferBindingSize and preparedElements * 2 <= device.limits.maxBufferSize and preparedBElements * 2 <= device.limits.maxStorageBufferBindingSize and preparedBElements * 2 <= device.limits.maxBufferSize", |
| "portableWorkgroupRows": "pow2ceil(min(16, max(4, pow2ceil(ceilDiv(precastRows, 4))), floor(deviceWorkgroupCap / portableWorkgroupCols), device.limits.maxComputeWorkgroupSizeY) + 1) / 2", |
| "portableTileM": "4 * portableWorkgroupRows", |
| "portableTileN": "4 * portableWorkgroupCols", |
| "portableTileK": 32, |
| "portablePartialK": "floor(16777216 / (255 * 255) / portableTileK) * portableTileK", |
| "portableGeometryOk": "portableWorkgroupRows >= 4 and portableWorkgroupCols <= device.limits.maxComputeWorkgroupSizeX and portablePartialK >= portableTileK and ceilDiv(precastRows, portableTileM) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", |
| "spatialAttributeLengthsOk": "kernelShapeLengthOk and stridesLengthOk and dilationsLengthOk and padsLengthOk", |
| "kernelShapeMatchesWeights": "not has(attrs, \"kernel_shape\") or (kernelW == kernelWidth and (spatialRank < 2 or kernelH == kernelHeight) and (spatialRank < 3 or kernelD == kernelDepth))", |
| "kernelExtentsOk": "kernelWidth >= 1 and (spatialRank < 2 or kernelHeight >= 1) and (spatialRank < 3 or kernelDepth >= 1)", |
| "stridesValuesOk": "not has(attrs, \"strides\") or (strideD >= 1 and floor(strideD) == strideD and strideH >= 1 and floor(strideH) == strideH and strideW >= 1 and floor(strideW) == strideW)", |
| "dilationsValuesOk": "not has(attrs, \"dilations\") or (dilationD >= 1 and floor(dilationD) == dilationD and dilationH >= 1 and floor(dilationH) == dilationH and dilationW >= 1 and floor(dilationW) == dilationW)", |
| "padsValuesOk": "padFront >= 0 and floor(padFront) == padFront and padTop >= 0 and floor(padTop) == padTop and padLeft >= 0 and floor(padLeft) == padLeft and padBack >= 0 and floor(padBack) == padBack and padBottom >= 0 and floor(padBottom) == padBottom and padRight >= 0 and floor(padRight) == padRight", |
| "explicitPadsOk": "attrs.auto_pad == \"NOTSET\" or not has(attrs, \"pads\")", |
| "spatialAttributesOk": "spatialRank >= 1 and spatialRank <= 3 and spatialAttributeLengthsOk and kernelShapeMatchesWeights and kernelExtentsOk and stridesValuesOk and dilationsValuesOk and padsValuesOk and explicitPadsOk", |
| "xyQuantContract": "(ranks.x_scale == 0 or (ranks.x_scale == 1 and dim(shapes.x_scale, 0) == 1)) and (ranks.y_scale == 0 or (ranks.y_scale == 1 and dim(shapes.y_scale, 0) == 1)) and (ranks.x_zero_point == 0 or (ranks.x_zero_point == 1 and dim(shapes.x_zero_point, 0) == 1)) and (ranks.y_zero_point == 0 or (ranks.y_zero_point == 1 and dim(shapes.y_zero_point, 0) == 1))", |
| "rank3TensorOk": "spatialAttributesOk and ranks.x == 3 and ranks.w == 3 and ranks.y == 3", |
| "rank5TensorOk": "spatialAttributesOk and ranks.x == 5 and ranks.w == 5 and ranks.y == 5", |
| "scalarWeightQuantOk": "(ranks.w_scale == 0 or (ranks.w_scale == 1 and dim(shapes.w_scale, 0) == 1)) and (ranks.w_zero_point == 0 or (ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == 1))", |
| "yScalar": "\"u32\" if tensorDtypes.y == \"uint8\" else \"i32\"", |
| "yUnsigned": "tensorDtypes.y == \"uint8\"", |
| "qMin": "0 if tensorDtypes.y == \"uint8\" else 0 - 128", |
| "qMax": "255 if tensorDtypes.y == \"uint8\" else 127", |
| "hasBZero": true, |
| "packedFeature": "device.wgslLanguageFeatures.has(\"packed_4x8_integer_dot_product\")", |
| "rank4TensorOk": "spatialAttributesOk and ranks.x == 4 and ranks.w == 4 and ranks.y == 4", |
| "outputBatchChannelsOk": "dim(shapes.y, 0) == dim(shapes.x, 0) and (dim(shapes.y, ranks.y - 1) if channelsLast else dim(shapes.y, 1)) == convOutChannels", |
| "groupChannelsOk": "attrs.group >= 1 and dim(shapes.w, 1) * attrs.group == convInChannels and convOutChannels % attrs.group == 0", |
| "outputSpatialOk": "outputWidth == expectedOutputWidth and (spatialRank < 2 or outputHeight == expectedOutputHeight) and (spatialRank < 3 or outputDepth == expectedOutputDepth)", |
| "perChannelWeightQuantOk": "ranks.w_scale == 1 and dim(shapes.w_scale, 0) == convOutChannels and ranks.w_zero_point == 1 and dim(shapes.w_zero_point, 0) == convOutChannels", |
| "flexibleWeightQuantOk": "scalarWeightQuantOk or perChannelWeightQuantOk", |
| "outImageSize": "1 if channelsLast else outSpatial", |
| "convKPadded": "ceilDiv(convK, 4) * 4", |
| "biasContractOk": "present.B and ranks.B == 1 and dim(shapes.B, 0) == convOutChannels", |
| "gemmDispatchOk": "ceilDiv(outSpatial, 64) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(convOutChannels, 64) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(convOutChannels, 8) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(outSpatial, 8) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and dim(shapes.x, 0) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", |
| "kMajorWeightStorageOk": "convKPadded * convOutChannels * 4 <= device.limits.maxStorageBufferBindingSize", |
| "nhwcColsStorageOk": "dim(shapes.x, 0) * outSpatial * convKPadded * 4 <= device.limits.maxStorageBufferBindingSize", |
| "nchwColsStorageOk": "dim(shapes.x, 0) * convK * outSpatial * 4 <= device.limits.maxStorageBufferBindingSize", |
| "aUnsigned": "tensorDtypes.x == \"uint8\" if channelsLast else tensorDtypes.w == \"uint8\"", |
| "bUnsigned": "tensorDtypes.w == \"uint8\" if channelsLast else tensorDtypes.x == \"uint8\"", |
| "aScalar": "\"u32\" if aUnsigned else \"i32\"", |
| "bScalar": "\"u32\" if bUnsigned else \"i32\"", |
| "xScalar": "\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\"", |
| "wScalar": "\"u32\" if tensorDtypes.w == \"uint8\" else \"i32\"", |
| "aZeroPerRow": "perChannelWeightQuantOk and not channelsLast", |
| "bZeroPerColumn": "perChannelWeightQuantOk and channelsLast", |
| "requant": "(\"per_column\" if channelsLast else \"per_row\") if perChannelWeightQuantOk else \"per_tensor\"", |
| "wZeroPerChannel": "perChannelWeightQuantOk", |
| "sgmatDispatchOk": "ceilDiv(sgmatN, sgmatTileN) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(sgmatM, sgmatTileM) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", |
| "sgmatFullGrid": "floor(sgmatM / sgmatTileM) * floor(sgmatN / sgmatTileN) * (1 if channelsLast else dim(shapes.x, 0))", |
| "sgmatUsefulInvocations": "sgmatFullGrid * sgmatWorkgroup * convK / (convK + 2 * sgmatTileK)", |
| "precastFullGrid": "floor(sgmatM / precastTileM) * floor(sgmatN / precastTileN) * (1 if channelsLast else dim(shapes.x, 0))", |
| "precastUsefulInvocations": "precastFullGrid * precastWorkgroup * convK / (convK + 2 * precastTileK)", |
| "precastAmortizedInvocations": "precastUsefulInvocations * max(1, (convK - 2 * precastTileK) / precastTileK)", |
| "precast_pointwise_nchw_ok": "(attrs.channels_last == 0) and (not present.B) and (rank4TensorOk) and (flexibleWeightQuantOk) and (attrs.group == 1) and (kernelHeight == 1) and (kernelWidth == 1) and (strideH == 1) and (strideW == 1) and (padTop == 0 and padBottom == 0) and (padLeft == 0 and padRight == 0) and (dim(shapes.w, 1) == convInChannels) and (outputHeight == inputHeight) and (outputWidth == inputWidth) and (gemmDispatchOk)", |
| "precast_im2col_nchw_ok": "(attrs.channels_last == 0) and (not present.B) and (rank4TensorOk) and (flexibleWeightQuantOk) and (attrs.group == 1) and (dim(shapes.w, 1) == convInChannels) and (outputSpatialOk) and ((kernelHeight > 1 or kernelWidth > 1)) and (convK >= 64) and (convK <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)) and (nchwColsStorageOk) and (gemmDispatchOk)", |
| "precast_im2col_nchw_bias_ok": "(attrs.channels_last == 0) and (biasContractOk) and (rank4TensorOk) and (flexibleWeightQuantOk) and (attrs.group == 1) and (dim(shapes.w, 1) == convInChannels) and (outputSpatialOk) and (convK <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)) and (nchwColsStorageOk) and (gemmDispatchOk)", |
| "precast_pointwise_nhwc_ok": "(attrs.channels_last != 0) and (not present.B) and (rank4TensorOk) and (flexibleWeightQuantOk) and (attrs.group == 1) and (dim(shapes.w, 1) == convInChannels) and (kernelHeight == 1) and (kernelWidth == 1) and (strideH == 1) and (strideW == 1) and (padTop == 0 and padBottom == 0) and (padLeft == 0 and padRight == 0) and (outputHeight == inputHeight) and (outputWidth == inputWidth) and (kMajorWeightStorageOk) and (gemmDispatchOk)", |
| "precast_pointwise_nhwc_bias_ok": "(attrs.channels_last != 0) and (biasContractOk) and (rank4TensorOk) and (flexibleWeightQuantOk) and (attrs.group == 1) and (dim(shapes.w, 1) == convInChannels) and (kernelHeight == 1) and (kernelWidth == 1) and (strideH == 1) and (strideW == 1) and (padTop == 0 and padBottom == 0) and (padLeft == 0 and padRight == 0) and (outputHeight == inputHeight) and (outputWidth == inputWidth) and (kMajorWeightStorageOk) and (gemmDispatchOk)", |
| "precast_im2col_nhwc_ok": "(attrs.channels_last != 0) and (not present.B) and (rank4TensorOk) and (flexibleWeightQuantOk) and (attrs.group == 1) and (dim(shapes.w, 1) == convInChannels) and ((kernelHeight > 1 or kernelWidth > 1)) and (outputSpatialOk) and (nhwcColsStorageOk) and (kMajorWeightStorageOk) and (gemmDispatchOk)", |
| "precast_im2col_nhwc_bias_ok": "(attrs.channels_last != 0) and (biasContractOk) and (rank4TensorOk) and (flexibleWeightQuantOk) and (attrs.group == 1) and (dim(shapes.w, 1) == convInChannels) and ((kernelHeight > 1 or kernelWidth > 1)) and (outputSpatialOk) and (nhwcColsStorageOk) and (kMajorWeightStorageOk) and (gemmDispatchOk)", |
| "portableLoadsOk": "portableTileM * portableTileK % (4 * portableWorkgroupRows * portableWorkgroupCols) == 0 and portableTileN * portableTileK % (4 * portableWorkgroupRows * portableWorkgroupCols) == 0", |
| "nhwcOc4UnitColumnThreads": "dim(shapes.y, 0) * outputHeight * outputWidth * ceilDiv(convOutChannels, 4)", |
| "nhwcOc4ColumnTile": "min(tunables.NHWC_OW_TILE, max(1, floor(nhwcOc4UnitColumnThreads / (tunables.WORKGROUP_SIZE * tunables.NHWC_BIAS_TARGET_WORKGROUPS)))) if present.B else tunables.NHWC_OW_TILE", |
| "nhwcOc4Threads": "dim(shapes.y, 0) * outputHeight * ceilDiv(outputWidth, nhwcOc4ColumnTile) * ceilDiv(convOutChannels, 4)", |
| "nhwcOc4Workgroups": "ceilDiv(nhwcOc4Threads, tunables.WORKGROUP_SIZE)", |
| "nhwcOc4DispatchOk": "min(nhwcOc4Workgroups, DISPATCH_FOLD_WIDTH) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535) and ceilDiv(nhwcOc4Workgroups, DISPATCH_FOLD_WIDTH) <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", |
| "bScalePerColumn": "perChannelWeightQuantOk and channelsLast", |
| "matrixBatchB": "not channelsLast", |
| "matrixBiasPerRow": "not channelsLast", |
| "matrixSubgroupId": true, |
| "prepareBatchB": "not channelsLast", |
| "requantize": true, |
| "wUnsigned": "tensorDtypes.w == \"uint8\"", |
| "xUnsigned": "tensorDtypes.x == \"uint8\"" |
| }, |
| "when": ["outputBatchChannelsOk", "xyQuantContract"], |
| "bindings": { |
| "x_w": { "name": "x", "elementType": "$xScalar" }, |
| "x_zero_point_w": { "name": "x_zero_point", "elementType": "$xScalar", "length": 1 }, |
| "w_w": { "name": "w", "elementType": "$wScalar" }, |
| "w_zero_point_w": { "name": "w_zero_point", "elementType": "$wScalar" }, |
| "x_scale": { "elementType": "f32", "length": 1 }, |
| "w_scale": { "elementType": "f32" }, |
| "y_scale": { "elementType": "f32", "length": 1 }, |
| "y_zero_point_w": { "name": "y_zero_point", "elementType": "$yScalar", "length": 1 }, |
| "y_w": { "name": "y", "elementType": "$yScalar" }, |
| "B": { "elementType": "i32" }, |
| "accum_out": { "scratch": "accum", "name": "y", "elementType": "i32" }, |
| "accum_in": { "scratch": "accum", "name": "accum", "buffer": "read-only-storage", "elementType": "i32" }, |
| "accum_params_2d": { |
| "name": "params", |
| "struct": [ |
| { "name": "inChannels", "type": "u32", "value": "convInChannels" }, |
| { "name": "inH", "type": "u32", "value": "inputHeight" }, |
| { "name": "inW", "type": "u32", "value": "inputWidth" }, |
| { "name": "outChannels", "type": "u32", "value": "convOutChannels" }, |
| { "name": "weightInChannels", "type": "u32", "value": "dim(shapes.w, 1)" }, |
| { "name": "inChannelsPerGroup", "type": "u32", "value": "convInChannels / attrs.group" }, |
| { "name": "outChannelsPerGroup", "type": "u32", "value": "convOutChannels / attrs.group" }, |
| { "name": "kernelH", "type": "u32", "value": "kernelHeight" }, |
| { "name": "kernelW", "type": "u32", "value": "kernelWidth" }, |
| { "name": "outH", "type": "u32", "value": "outputHeight" }, |
| { "name": "outW", "type": "u32", "value": "outputWidth" }, |
| { "name": "strideH", "type": "u32", "value": "strideH" }, |
| { "name": "strideW", "type": "u32", "value": "strideW" }, |
| { "name": "dilationH", "type": "u32", "value": "dilationH" }, |
| { "name": "dilationW", "type": "u32", "value": "dilationW" }, |
| { "name": "padH", "type": "i32", "value": "effectivePadTop" }, |
| { "name": "padW", "type": "i32", "value": "effectivePadLeft" }, |
| { "name": "count", "type": "u32", "value": "numel(shapes.y)" } |
| ] |
| }, |
| "accum_params_3d": { |
| "name": "params", |
| "struct": [ |
| { "name": "inChannels", "type": "u32", "value": "convInChannels" }, |
| { "name": "inD", "type": "u32", "value": "inputDepth" }, |
| { "name": "inH", "type": "u32", "value": "inputHeight" }, |
| { "name": "inW", "type": "u32", "value": "inputWidth" }, |
| { "name": "outChannels", "type": "u32", "value": "convOutChannels" }, |
| { "name": "weightInChannels", "type": "u32", "value": "dim(shapes.w, 1)" }, |
| { "name": "inChannelsPerGroup", "type": "u32", "value": "convInChannels / attrs.group" }, |
| { "name": "outChannelsPerGroup", "type": "u32", "value": "convOutChannels / attrs.group" }, |
| { "name": "kernelD", "type": "u32", "value": "kernelDepth" }, |
| { "name": "kernelH", "type": "u32", "value": "kernelHeight" }, |
| { "name": "kernelW", "type": "u32", "value": "kernelWidth" }, |
| { "name": "outD", "type": "u32", "value": "outputDepth" }, |
| { "name": "outH", "type": "u32", "value": "outputHeight" }, |
| { "name": "outW", "type": "u32", "value": "outputWidth" }, |
| { "name": "strideD", "type": "u32", "value": "strideD" }, |
| { "name": "strideH", "type": "u32", "value": "strideH" }, |
| { "name": "strideW", "type": "u32", "value": "strideW" }, |
| { "name": "dilationD", "type": "u32", "value": "dilationD" }, |
| { "name": "dilationH", "type": "u32", "value": "dilationH" }, |
| { "name": "dilationW", "type": "u32", "value": "dilationW" }, |
| { "name": "padD", "type": "i32", "value": "effectivePadFront" }, |
| { "name": "padH", "type": "i32", "value": "effectivePadTop" }, |
| { "name": "padW", "type": "i32", "value": "effectivePadLeft" }, |
| { "name": "count", "type": "u32", "value": "numel(shapes.y)" } |
| ] |
| }, |
| "requant_params": { |
| "name": "params", |
| "struct": [ |
| { "name": "count", "type": "u32", "value": "numel(shapes.y)" }, |
| { "name": "outChannels", "type": "u32", "value": "convOutChannels" }, |
| { "name": "outImageSize", "type": "u32", "value": "outImageSize" } |
| ] |
| }, |
| "requant_params_count": { |
| "name": "params", |
| "struct": [{ "name": "count", "type": "u32", "value": "numel(shapes.y)" }] |
| }, |
| "x4_params": { |
| "name": "params", |
| "struct": [ |
| { "name": "batch", "type": "u32", "value": "dim(shapes.x, 0)" }, |
| { "name": "inChannels", "type": "u32", "value": "convInChannels" }, |
| { "name": "inH", "type": "u32", "value": "inputHeight" }, |
| { "name": "inW", "type": "u32", "value": "inputWidth" }, |
| { "name": "outChannels", "type": "u32", "value": "convOutChannels" }, |
| { "name": "weightInChannels", "type": "u32", "value": "dim(shapes.w, 1)" }, |
| { "name": "inChannelsPerGroup", "type": "u32", "value": "convInChannels / attrs.group" }, |
| { "name": "outChannelsPerGroup", "type": "u32", "value": "convOutChannels / attrs.group" }, |
| { "name": "kernelH", "type": "u32", "value": "kernelHeight" }, |
| { "name": "kernelW", "type": "u32", "value": "kernelWidth" }, |
| { "name": "outH", "type": "u32", "value": "outputHeight" }, |
| { "name": "outW", "type": "u32", "value": "outputWidth" }, |
| { "name": "strideH", "type": "u32", "value": "strideH" }, |
| { "name": "strideW", "type": "u32", "value": "strideW" }, |
| { "name": "dilationH", "type": "u32", "value": "dilationH" }, |
| { "name": "dilationW", "type": "u32", "value": "dilationW" }, |
| { "name": "padH", "type": "i32", "value": "effectivePadTop" }, |
| { "name": "padW", "type": "i32", "value": "effectivePadLeft" } |
| ] |
| }, |
| "oc4_params": { |
| "name": "params", |
| "struct": [ |
| { |
| "name": "count", |
| "type": "u32", |
| "value": "dim(shapes.x, 0) * outputHeight * ceilDiv(outputWidth, nhwcOc4ColumnTile) * ceilDiv(convOutChannels, 4)" |
| }, |
| { "name": "ocBlocks", "type": "u32", "value": "ceilDiv(convOutChannels, 4)" }, |
| { "name": "owGroups", "type": "u32", "value": "ceilDiv(outputWidth, nhwcOc4ColumnTile)" }, |
| { "name": "inChannels", "type": "u32", "value": "convInChannels" }, |
| { "name": "inH", "type": "u32", "value": "inputHeight" }, |
| { "name": "inW", "type": "u32", "value": "inputWidth" }, |
| { "name": "outChannels", "type": "u32", "value": "convOutChannels" }, |
| { "name": "weightInChannels", "type": "u32", "value": "dim(shapes.w, 1)" }, |
| { "name": "inChannelsPerGroup", "type": "u32", "value": "convInChannels / attrs.group" }, |
| { "name": "outChannelsPerGroup", "type": "u32", "value": "convOutChannels / attrs.group" }, |
| { "name": "kernelH", "type": "u32", "value": "kernelHeight" }, |
| { "name": "kernelW", "type": "u32", "value": "kernelWidth" }, |
| { "name": "outH", "type": "u32", "value": "outputHeight" }, |
| { "name": "outW", "type": "u32", "value": "outputWidth" }, |
| { "name": "strideH", "type": "u32", "value": "strideH" }, |
| { "name": "strideW", "type": "u32", "value": "strideW" }, |
| { "name": "dilationH", "type": "u32", "value": "dilationH" }, |
| { "name": "dilationW", "type": "u32", "value": "dilationW" }, |
| { "name": "padH", "type": "i32", "value": "effectivePadTop" }, |
| { "name": "padW", "type": "i32", "value": "effectivePadLeft" } |
| ] |
| }, |
| "a_weight": { "arg": "w", "name": "a", "elementType": "$aVec4" }, |
| "a_zero_point_weight": { "arg": "w_zero_point", "name": "a_zero_point", "elementType": "$aScalar" }, |
| "b_x": { "arg": "x", "name": "b", "elementType": "$bScalar" }, |
| "b_zero_point_x": { "arg": "x_zero_point", "name": "b_zero_point", "elementType": "$bScalar", "length": 1 }, |
| "b_cols_nchw": { "scratch": "cols", "name": "b", "buffer": "read-only-storage", "elementType": "$bScalar" }, |
| "a_x": { "arg": "x", "name": "a", "elementType": "$aVec4" }, |
| "a_cols_nhwc": { "scratch": "cols", "name": "a", "buffer": "read-only-storage", "elementType": "$aVec4" }, |
| "a_zero_point_x": { "arg": "x_zero_point", "name": "a_zero_point", "elementType": "$aScalar", "length": 1 }, |
| "b_weight_k_major": { |
| "scratch": "w_k_major", |
| "name": "b", |
| "buffer": "read-only-storage", |
| "elementType": "$bScalar" |
| }, |
| "b_zero_point_weight": { "arg": "w_zero_point", "name": "b_zero_point", "elementType": "$bScalar" }, |
| "a_scale_x": { "arg": "x_scale", "name": "a_scale", "elementType": "f32", "length": 1 }, |
| "b_scale_w": { "arg": "w_scale", "name": "b_scale", "elementType": "f32" }, |
| "bias": { "arg": "B", "name": "bias", "elementType": "i32" }, |
| "w_k_major_out": { "scratch": "w_k_major", "name": "w_k_major", "elementType": "$wScalar" }, |
| "k_major_params": { |
| "name": "params", |
| "struct": [ |
| { "name": "count", "type": "u32", "value": "convKPadded * convOutChannels" }, |
| { "name": "outChannels", "type": "u32", "value": "convOutChannels" }, |
| { "name": "weightInChannels", "type": "u32", "value": "dim(shapes.w, 1)" }, |
| { "name": "kernelH", "type": "u32", "value": "kernelHeight" }, |
| { "name": "kernelW", "type": "u32", "value": "kernelWidth" }, |
| { "name": "kRows", "type": "u32", "value": "convK" } |
| ] |
| }, |
| "k_major_params_exact": { |
| "name": "params", |
| "struct": [ |
| { "name": "count", "type": "u32", "value": "convK * convOutChannels" }, |
| { "name": "outChannels", "type": "u32", "value": "convOutChannels" }, |
| { "name": "weightInChannels", "type": "u32", "value": "dim(shapes.w, 1)" }, |
| { "name": "kernelH", "type": "u32", "value": "kernelHeight" }, |
| { "name": "kernelW", "type": "u32", "value": "kernelWidth" }, |
| { "name": "kRows", "type": "u32", "value": "convK" } |
| ] |
| }, |
| "x_im2col": { "name": "x", "elementType": "$xVec4" }, |
| "cols_out_vec4": { "scratch": "cols", "name": "cols", "elementType": "$colsVec4" }, |
| "im2col_nhwc_params": { |
| "name": "params", |
| "struct": [ |
| { |
| "name": "count", |
| "type": "u32", |
| "value": "ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)" |
| }, |
| { "name": "totalWords", "type": "u32", "value": "dim(shapes.x, 0) * outSpatial * (convKPadded / 4)" }, |
| { "name": "wordsPerRow", "type": "u32", "value": "convKPadded / 4" }, |
| { "name": "kRows", "type": "u32", "value": "convK" }, |
| { "name": "outCount", "type": "u32", "value": "outSpatial" }, |
| { "name": "inChannels", "type": "u32", "value": "convInChannels" }, |
| { "name": "inH", "type": "u32", "value": "inputHeight" }, |
| { "name": "inW", "type": "u32", "value": "inputWidth" }, |
| { "name": "outW", "type": "u32", "value": "outputWidth" } |
| ] |
| }, |
| "im2col_nchw_params": { |
| "name": "params", |
| "struct": [ |
| { "name": "outCount", "type": "u32", "value": "outSpatial" }, |
| { "name": "inChannels", "type": "u32", "value": "convInChannels" }, |
| { "name": "inH", "type": "u32", "value": "inputHeight" }, |
| { "name": "inW", "type": "u32", "value": "inputWidth" }, |
| { "name": "kRows", "type": "u32", "value": "convK" }, |
| { "name": "outW", "type": "u32", "value": "outputWidth" } |
| ] |
| }, |
| "gemm_params_nchw": { |
| "name": "params", |
| "struct": [ |
| { "name": "M", "type": "u32", "value": "convOutChannels" }, |
| { "name": "N", "type": "u32", "value": "outSpatial" }, |
| { "name": "K", "type": "u32", "value": "convK" }, |
| { "name": "aBatchStride4", "type": "u32", "value": 0 }, |
| { "name": "bBatchStride", "type": "u32", "value": "convK * outSpatial" }, |
| { "name": "yBatchStride", "type": "u32", "value": "convOutChannels * outSpatial" } |
| ] |
| }, |
| "gemm_params_nhwc_pointwise": { |
| "name": "params", |
| "struct": [ |
| { "name": "M", "type": "u32", "value": "outSpatial" }, |
| { "name": "N", "type": "u32", "value": "convOutChannels" }, |
| { "name": "K", "type": "u32", "value": "convK" }, |
| { |
| "name": "aBatchStride4", |
| "type": "u32", |
| "value": "(outSpatial * convK / 4) if convK % 4 == 0 else (outSpatial * convK)" |
| }, |
| { "name": "bBatchStride", "type": "u32", "value": 0 }, |
| { "name": "yBatchStride", "type": "u32", "value": "convOutChannels * outSpatial" } |
| ] |
| }, |
| "gemm_params_nhwc_im2col": { |
| "name": "params", |
| "struct": [ |
| { "name": "M", "type": "u32", "value": "outSpatial" }, |
| { "name": "N", "type": "u32", "value": "convOutChannels" }, |
| { "name": "K", "type": "u32", "value": "convKPadded" }, |
| { "name": "aBatchStride4", "type": "u32", "value": "outSpatial * convKPadded / 4" }, |
| { "name": "bBatchStride", "type": "u32", "value": 0 }, |
| { "name": "yBatchStride", "type": "u32", "value": "convOutChannels * outSpatial" } |
| ] |
| }, |
| "x_b": { "name": "x", "elementType": "$bScalar" }, |
| "x_zero_point_b": { "name": "x_zero_point", "elementType": "$bScalar", "length": 1 }, |
| "cols_out_b": { "scratch": "cols", "name": "cols", "elementType": "$bScalar" }, |
| "gemm_params_nchw_matrix": { |
| "name": "params", |
| "struct": [ |
| { "name": "M", "type": "u32", "value": "sgmatM" }, |
| { "name": "N", "type": "u32", "value": "outSpatial" }, |
| { "name": "K", "type": "u32", "value": "convK" }, |
| { "name": "bBatchStride", "type": "u32", "value": "convK * outSpatial" }, |
| { "name": "yBatchStride", "type": "u32", "value": "convOutChannels * outSpatial" } |
| ] |
| }, |
| "gemm_params_nhwc_pointwise_matrix": { |
| "name": "params", |
| "struct": [ |
| { "name": "M", "type": "u32", "value": "sgmatM" }, |
| { "name": "N", "type": "u32", "value": "convOutChannels" }, |
| { "name": "K", "type": "u32", "value": "convK" } |
| ] |
| }, |
| "gemm_params_nhwc_im2col_matrix": { |
| "name": "params", |
| "struct": [ |
| { "name": "M", "type": "u32", "value": "sgmatM" }, |
| { "name": "N", "type": "u32", "value": "convOutChannels" }, |
| { "name": "K", "type": "u32", "value": "convKPadded" } |
| ] |
| }, |
| "precast_aOut": { "scratch": "aFloat", "name": "aFloat", "elementType": "$precastScalar" }, |
| "precast_bOut": { "scratch": "bFloat", "name": "bFloat", "elementType": "$precastScalar" }, |
| "precast_a": { "scratch": "aFloat", "name": "a", "buffer": "read-only-storage", "elementType": "$precastScalar" }, |
| "precast_b": { "scratch": "bFloat", "name": "b", "buffer": "read-only-storage", "elementType": "$precastScalar" }, |
| "precast_matrix_params": { |
| "name": "params", |
| "struct": [ |
| { "name": "M", "type": "u32", "value": "convOutChannels" }, |
| { "name": "N", "type": "u32", "value": "outSpatial" }, |
| { "name": "K", "type": "u32", "value": "convK" }, |
| { "name": "yBatchStride", "type": "u32", "value": "convOutChannels * outSpatial" } |
| ] |
| }, |
| "precast_prepare_plain": { |
| "name": "params", |
| "struct": [ |
| { "name": "M", "type": "u32", "value": "sgmatM" }, |
| { "name": "K", "type": "u32", "value": "convK" }, |
| { "name": "paddedK", "type": "u32", "value": "preparedK" }, |
| { "name": "count", "type": "u32", "value": "prepareTotalElements" }, |
| { |
| "name": "groupsX", |
| "type": "u32", |
| "value": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| }, |
| { "name": "N", "type": "u32", "value": "sgmatN" }, |
| { "name": "paddedN", "type": "u32", "value": "preparedN" }, |
| { "name": "aCount", "type": "u32", "value": "preparedElements" } |
| ] |
| }, |
| "precast_prepare_spatial": { |
| "name": "params", |
| "struct": [ |
| { "name": "M", "type": "u32", "value": "sgmatM" }, |
| { "name": "K", "type": "u32", "value": "convK" }, |
| { "name": "paddedK", "type": "u32", "value": "preparedK" }, |
| { "name": "count", "type": "u32", "value": "prepareTotalElements" }, |
| { |
| "name": "groupsX", |
| "type": "u32", |
| "value": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| }, |
| { "name": "N", "type": "u32", "value": "sgmatN" }, |
| { "name": "paddedN", "type": "u32", "value": "preparedN" }, |
| { "name": "aCount", "type": "u32", "value": "preparedElements" }, |
| { "name": "inChannels", "type": "u32", "value": "convInChannels" }, |
| { "name": "inH", "type": "u32", "value": "inputHeight" }, |
| { "name": "inW", "type": "u32", "value": "inputWidth" }, |
| { "name": "outW", "type": "u32", "value": "outputWidth" } |
| ] |
| }, |
| "precast_matrix_nhwc": { |
| "name": "params", |
| "struct": [ |
| { "name": "M", "type": "u32", "value": "sgmatM" }, |
| { "name": "N", "type": "u32", "value": "sgmatN" }, |
| { "name": "K", "type": "u32", "value": "convK" } |
| ] |
| }, |
| "precast_weight_oihw": { "arg": "w", "name": "b", "elementType": "$bScalar" }, |
| "precast_prepare_nhwc_plain": { |
| "name": "params", |
| "struct": [ |
| { "name": "M", "type": "u32", "value": "sgmatM" }, |
| { "name": "K", "type": "u32", "value": "convK" }, |
| { "name": "paddedK", "type": "u32", "value": "preparedK" }, |
| { "name": "count", "type": "u32", "value": "prepareTotalElements" }, |
| { |
| "name": "groupsX", |
| "type": "u32", |
| "value": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| }, |
| { "name": "N", "type": "u32", "value": "sgmatN" }, |
| { "name": "paddedN", "type": "u32", "value": "preparedN" }, |
| { "name": "aCount", "type": "u32", "value": "preparedElements" } |
| ] |
| }, |
| "precast_prepare_nhwc_spatial": { |
| "name": "params", |
| "struct": [ |
| { "name": "M", "type": "u32", "value": "sgmatM" }, |
| { "name": "K", "type": "u32", "value": "convK" }, |
| { "name": "paddedK", "type": "u32", "value": "preparedK" }, |
| { "name": "count", "type": "u32", "value": "prepareTotalElements" }, |
| { |
| "name": "groupsX", |
| "type": "u32", |
| "value": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| }, |
| { "name": "N", "type": "u32", "value": "sgmatN" }, |
| { "name": "paddedN", "type": "u32", "value": "preparedN" }, |
| { "name": "aCount", "type": "u32", "value": "preparedElements" }, |
| { "name": "outCount", "type": "u32", "value": "outSpatial" }, |
| { "name": "inChannels", "type": "u32", "value": "convInChannels" }, |
| { "name": "inH", "type": "u32", "value": "inputHeight" }, |
| { "name": "inW", "type": "u32", "value": "inputWidth" }, |
| { "name": "outW", "type": "u32", "value": "outputWidth" } |
| ] |
| }, |
| "portable_f16_a": { "name": "a", "scratch": "aFloat", "buffer": "read-only-storage", "elementType": "vec4<f16>" }, |
| "portable_f16_b": { "name": "b", "scratch": "bFloat", "buffer": "read-only-storage", "elementType": "vec4<f16>" }, |
| "portable_f32_a": { "name": "a", "scratch": "aFloat", "buffer": "read-only-storage", "elementType": "vec4<f32>" }, |
| "portable_f32_b": { "name": "b", "scratch": "bFloat", "buffer": "read-only-storage", "elementType": "vec4<f32>" } |
| }, |
| "variants": [ |
| { |
| "id": "spatial_nd2_scalar", |
| "when": ["(rank3TensorOk or rank4TensorOk)", "scalarWeightQuantOk", "not present.B", "groupChannelsOk", "outputSpatialOk"], |
| "derive": { "hasBias": false, "wScalePerChannel": false, "wZeroPerChannel": false }, |
| "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }], |
| "passes": [ |
| { |
| "id": "accumulate", |
| "name": "QLinearConv.Accumulate", |
| "shader": "conv-int-accumulate-spatial.wgsl.jinja", |
| "derive": { "spatialRankSpec": 2 }, |
| "bindings": ["x_w", "x_zero_point_w", "w_w", "w_zero_point_w", "accum_out", "accum_params_2d"], |
| "dispatch": { |
| "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| }, |
| { |
| "id": "requantize", |
| "name": "QLinearConv.Requantize", |
| "shader": "qlinear-conv-requantize.wgsl.jinja", |
| "bindings": ["accum_in", "x_scale", "w_scale", "y_scale", "y_zero_point_w", "y_w", "requant_params_count"], |
| "dispatch": { |
| "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| } |
| ] |
| }, |
| { |
| "id": "spatial_nd2_scalar_bias", |
| "when": ["(rank3TensorOk or rank4TensorOk)", "scalarWeightQuantOk", "biasContractOk", "groupChannelsOk", "outputSpatialOk"], |
| "derive": { "hasBias": true, "wScalePerChannel": false, "wZeroPerChannel": false }, |
| "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }], |
| "passes": [ |
| { |
| "id": "accumulate", |
| "name": "QLinearConv.Accumulate", |
| "shader": "conv-int-accumulate-spatial.wgsl.jinja", |
| "derive": { "spatialRankSpec": 2 }, |
| "bindings": ["x_w", "x_zero_point_w", "w_w", "w_zero_point_w", "accum_out", "accum_params_2d"], |
| "dispatch": { |
| "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| }, |
| { |
| "id": "requantize", |
| "name": "QLinearConv.Requantize", |
| "shader": "qlinear-conv-requantize.wgsl.jinja", |
| "bindings": ["accum_in", "x_scale", "w_scale", "y_scale", "y_zero_point_w", "B", "y_w", "requant_params"], |
| "dispatch": { |
| "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| } |
| ] |
| }, |
| { |
| "id": "spatial_nd2_per_channel", |
| "when": ["(rank3TensorOk or rank4TensorOk)", "perChannelWeightQuantOk", "not present.B", "groupChannelsOk", "outputSpatialOk"], |
| "derive": { "hasBias": false, "wScalePerChannel": true, "wZeroPerChannel": true }, |
| "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }], |
| "passes": [ |
| { |
| "id": "accumulate", |
| "name": "QLinearConv.Accumulate", |
| "shader": "conv-int-accumulate-spatial.wgsl.jinja", |
| "derive": { "spatialRankSpec": 2 }, |
| "bindings": ["x_w", "x_zero_point_w", "w_w", "w_zero_point_w", "accum_out", "accum_params_2d"], |
| "dispatch": { |
| "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| }, |
| { |
| "id": "requantize", |
| "name": "QLinearConv.Requantize", |
| "shader": "qlinear-conv-requantize.wgsl.jinja", |
| "bindings": ["accum_in", "x_scale", "w_scale", "y_scale", "y_zero_point_w", "y_w", "requant_params"], |
| "dispatch": { |
| "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| } |
| ] |
| }, |
| { |
| "id": "spatial_nd2_per_channel_bias", |
| "when": ["(rank3TensorOk or rank4TensorOk)", "perChannelWeightQuantOk", "biasContractOk", "groupChannelsOk", "outputSpatialOk"], |
| "derive": { "hasBias": true, "wScalePerChannel": true, "wZeroPerChannel": true }, |
| "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }], |
| "passes": [ |
| { |
| "id": "accumulate", |
| "name": "QLinearConv.Accumulate", |
| "shader": "conv-int-accumulate-spatial.wgsl.jinja", |
| "derive": { "spatialRankSpec": 2 }, |
| "bindings": ["x_w", "x_zero_point_w", "w_w", "w_zero_point_w", "accum_out", "accum_params_2d"], |
| "dispatch": { |
| "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| }, |
| { |
| "id": "requantize", |
| "name": "QLinearConv.Requantize", |
| "shader": "qlinear-conv-requantize.wgsl.jinja", |
| "bindings": ["accum_in", "x_scale", "w_scale", "y_scale", "y_zero_point_w", "B", "y_w", "requant_params"], |
| "dispatch": { |
| "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| } |
| ] |
| }, |
| { |
| "id": "spatial_nd3_scalar", |
| "when": ["rank5TensorOk", "scalarWeightQuantOk", "not present.B", "groupChannelsOk", "outputSpatialOk"], |
| "derive": { "hasBias": false, "wScalePerChannel": false, "wZeroPerChannel": false }, |
| "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }], |
| "passes": [ |
| { |
| "id": "accumulate", |
| "name": "QLinearConv.Accumulate3d", |
| "shader": "conv-int-accumulate-spatial.wgsl.jinja", |
| "derive": { "spatialRankSpec": 3 }, |
| "bindings": ["x_w", "x_zero_point_w", "w_w", "w_zero_point_w", "accum_out", "accum_params_3d"], |
| "dispatch": { |
| "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| }, |
| { |
| "id": "requantize", |
| "name": "QLinearConv.Requantize", |
| "shader": "qlinear-conv-requantize.wgsl.jinja", |
| "bindings": ["accum_in", "x_scale", "w_scale", "y_scale", "y_zero_point_w", "y_w", "requant_params_count"], |
| "dispatch": { |
| "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| } |
| ] |
| }, |
| { |
| "id": "spatial_nd3_scalar_bias", |
| "when": ["rank5TensorOk", "scalarWeightQuantOk", "biasContractOk", "groupChannelsOk", "outputSpatialOk"], |
| "derive": { "hasBias": true, "wScalePerChannel": false, "wZeroPerChannel": false }, |
| "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }], |
| "passes": [ |
| { |
| "id": "accumulate", |
| "name": "QLinearConv.Accumulate3d", |
| "shader": "conv-int-accumulate-spatial.wgsl.jinja", |
| "derive": { "spatialRankSpec": 3 }, |
| "bindings": ["x_w", "x_zero_point_w", "w_w", "w_zero_point_w", "accum_out", "accum_params_3d"], |
| "dispatch": { |
| "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| }, |
| { |
| "id": "requantize", |
| "name": "QLinearConv.Requantize", |
| "shader": "qlinear-conv-requantize.wgsl.jinja", |
| "bindings": ["accum_in", "x_scale", "w_scale", "y_scale", "y_zero_point_w", "B", "y_w", "requant_params"], |
| "dispatch": { |
| "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| } |
| ] |
| }, |
| { |
| "id": "spatial_nd3_per_channel", |
| "when": ["rank5TensorOk", "perChannelWeightQuantOk", "not present.B", "groupChannelsOk", "outputSpatialOk"], |
| "derive": { "hasBias": false, "wScalePerChannel": true, "wZeroPerChannel": true }, |
| "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }], |
| "passes": [ |
| { |
| "id": "accumulate", |
| "name": "QLinearConv.Accumulate3d", |
| "shader": "conv-int-accumulate-spatial.wgsl.jinja", |
| "derive": { "spatialRankSpec": 3 }, |
| "bindings": ["x_w", "x_zero_point_w", "w_w", "w_zero_point_w", "accum_out", "accum_params_3d"], |
| "dispatch": { |
| "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| }, |
| { |
| "id": "requantize", |
| "name": "QLinearConv.Requantize", |
| "shader": "qlinear-conv-requantize.wgsl.jinja", |
| "bindings": ["accum_in", "x_scale", "w_scale", "y_scale", "y_zero_point_w", "y_w", "requant_params"], |
| "dispatch": { |
| "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| } |
| ] |
| }, |
| { |
| "id": "spatial_nd3_per_channel_bias", |
| "when": ["rank5TensorOk", "perChannelWeightQuantOk", "biasContractOk", "groupChannelsOk", "outputSpatialOk"], |
| "derive": { "hasBias": true, "wScalePerChannel": true, "wZeroPerChannel": true }, |
| "intermediates": [{ "id": "accum", "dtype": "int32", "shape": "shapes.y" }], |
| "passes": [ |
| { |
| "id": "accumulate", |
| "name": "QLinearConv.Accumulate3d", |
| "shader": "conv-int-accumulate-spatial.wgsl.jinja", |
| "derive": { "spatialRankSpec": 3 }, |
| "bindings": ["x_w", "x_zero_point_w", "w_w", "w_zero_point_w", "accum_out", "accum_params_3d"], |
| "dispatch": { |
| "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| }, |
| { |
| "id": "requantize", |
| "name": "QLinearConv.Requantize", |
| "shader": "qlinear-conv-requantize.wgsl.jinja", |
| "bindings": ["accum_in", "x_scale", "w_scale", "y_scale", "y_zero_point_w", "B", "y_w", "requant_params"], |
| "dispatch": { |
| "x": "min(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((numel(shapes.y)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| } |
| ] |
| }, |
| { |
| "id": "nchw2d_columns_x4", |
| "priority": 10, |
| "when": ["attrs.channels_last == 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "groupChannelsOk", "outputSpatialOk", "outputWidth >= 4", "convK >= 4", "numel(shapes.y) >= 32"], |
| "derive": { |
| "wScalePerChannel": "perChannelWeightQuantOk", |
| "ocTile": "4 if convOutChannels >= attrs.group * 4 and convOutChannels % (attrs.group * 4) == 0 else 1" |
| }, |
| "passes": [ |
| { |
| "id": "main", |
| "name": "QLinearConv.ColumnsX4", |
| "shader": "qlinear-conv-nchw-x4.wgsl.jinja", |
| "bindings": ["x_w", "x_scale", "x_zero_point_w", "w_w", "w_scale", "w_zero_point_w", "y_scale", "y_zero_point_w", "y_w", "x4_params"], |
| "dispatch": { |
| "x": "min(ceilDiv((dim(shapes.y, 0) * ceilDiv(convOutChannels, 4 if convOutChannels >= attrs.group * 4 and convOutChannels % (attrs.group * 4) == 0 else 1) * outputHeight * ceilDiv(outputWidth, 4)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((dim(shapes.y, 0) * ceilDiv(convOutChannels, 4 if convOutChannels >= attrs.group * 4 and convOutChannels % (attrs.group * 4) == 0 else 1) * outputHeight * ceilDiv(outputWidth, 4)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| } |
| ] |
| }, |
| { |
| "id": "nhwc2d_channels_x4", |
| "priority": 10, |
| "when": ["attrs.channels_last != 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "groupChannelsOk", "outputSpatialOk", "convOutChannels % (attrs.group * 4) == 0", "numel(shapes.y) >= 32"], |
| "derive": { |
| "wScalePerChannel": "perChannelWeightQuantOk", |
| "depthwise": false, |
| "owTile": "nhwcOc4ColumnTile", |
| "hasBias": false |
| }, |
| "passes": [ |
| { |
| "id": "main", |
| "name": "QLinearConv.ChannelsX4", |
| "shader": "qlinear-conv-nhwc-oc4.wgsl.jinja", |
| "bindings": ["x_w", "x_scale", "x_zero_point_w", "w_w", "w_scale", "w_zero_point_w", "y_scale", "y_zero_point_w", "y_w", "oc4_params"], |
| "dispatch": { |
| "x": "min(ceilDiv((nhwcOc4Threads), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((nhwcOc4Threads), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| } |
| ] |
| }, |
| { |
| "id": "nhwc2d_depthwise_x4", |
| "priority": 10, |
| "when": ["attrs.channels_last != 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "groupChannelsOk", "outputSpatialOk", "attrs.group == convInChannels", "attrs.group == convOutChannels", "convOutChannels % 4 == 0", "numel(shapes.y) >= 32"], |
| "derive": { |
| "wScalePerChannel": "perChannelWeightQuantOk", |
| "depthwise": true, |
| "owTile": "nhwcOc4ColumnTile", |
| "hasBias": false |
| }, |
| "passes": [ |
| { |
| "id": "main", |
| "name": "QLinearConv.ChannelsX4", |
| "shader": "qlinear-conv-nhwc-oc4.wgsl.jinja", |
| "bindings": ["x_w", "x_scale", "x_zero_point_w", "w_w", "w_scale", "w_zero_point_w", "y_scale", "y_zero_point_w", "y_w", "oc4_params"], |
| "dispatch": { |
| "x": "min(ceilDiv((nhwcOc4Threads), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((nhwcOc4Threads), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| } |
| ] |
| }, |
| { |
| "id": "nhwc2d_channels_x4_bias", |
| "priority": 10, |
| "when": ["attrs.channels_last != 0", "biasContractOk", "rank4TensorOk", "flexibleWeightQuantOk", "groupChannelsOk", "outputSpatialOk", "convOutChannels % (attrs.group * 4) == 0", "numel(shapes.y) >= 32", "attrs.group > 1", "device.limits.maxStorageBuffersPerShaderStage >= 10", "tunables.WORKGROUP_SIZE <= deviceWorkgroupCap", "tunables.WORKGROUP_SIZE <= device.limits.maxComputeWorkgroupSizeX", "nhwcOc4DispatchOk"], |
| "derive": { |
| "wScalePerChannel": "perChannelWeightQuantOk", |
| "depthwise": false, |
| "owTile": "nhwcOc4ColumnTile", |
| "hasBias": true |
| }, |
| "passes": [ |
| { |
| "id": "main", |
| "name": "QLinearConv.ChannelsX4", |
| "shader": "qlinear-conv-nhwc-oc4.wgsl.jinja", |
| "bindings": ["x_w", "x_scale", "x_zero_point_w", "w_w", "w_scale", "w_zero_point_w", "y_scale", "y_zero_point_w", "y_w", "oc4_params", "bias"], |
| "dispatch": { |
| "x": "min(ceilDiv((nhwcOc4Threads), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((nhwcOc4Threads), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| } |
| ] |
| }, |
| { |
| "id": "nhwc2d_depthwise_x4_bias", |
| "priority": 10, |
| "when": ["attrs.channels_last != 0", "biasContractOk", "rank4TensorOk", "flexibleWeightQuantOk", "groupChannelsOk", "outputSpatialOk", "attrs.group == convInChannels", "attrs.group == convOutChannels", "convOutChannels % 4 == 0", "numel(shapes.y) >= 32", "attrs.group > 1", "device.limits.maxStorageBuffersPerShaderStage >= 10", "tunables.WORKGROUP_SIZE <= deviceWorkgroupCap", "tunables.WORKGROUP_SIZE <= device.limits.maxComputeWorkgroupSizeX", "nhwcOc4DispatchOk"], |
| "derive": { |
| "wScalePerChannel": "perChannelWeightQuantOk", |
| "depthwise": true, |
| "owTile": "nhwcOc4ColumnTile", |
| "hasBias": true |
| }, |
| "passes": [ |
| { |
| "id": "main", |
| "name": "QLinearConv.ChannelsX4", |
| "shader": "qlinear-conv-nhwc-oc4.wgsl.jinja", |
| "bindings": ["x_w", "x_scale", "x_zero_point_w", "w_w", "w_scale", "w_zero_point_w", "y_scale", "y_zero_point_w", "y_w", "oc4_params", "bias"], |
| "dispatch": { |
| "x": "min(ceilDiv((nhwcOc4Threads), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((nhwcOc4Threads), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| } |
| ] |
| }, |
| { |
| "id": "dp4a_pointwise_nchw", |
| "priority": 20, |
| "when": ["packedFeature", "attrs.channels_last == 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "kernelHeight == 1", "kernelWidth == 1", "strideH == 1", "strideW == 1", "padTop == 0 and padBottom == 0", "padLeft == 0 and padRight == 0", "dim(shapes.w, 1) == convInChannels", "outputHeight == inputHeight", "outputWidth == inputWidth", "gemmDispatchOk"], |
| "derive": { |
| "aPacked": "convK % 4 == 0", |
| "aVec4": "(\"vec4<u32>\" if aUnsigned else \"vec4<i32>\") if aPacked else aScalar", |
| "hasBias": false, |
| "biasPerColumn": "channelsLast", |
| "tileM": "4 if dp4aAppleCompact else 8", |
| "tileN": 64, |
| "ktWords": "dp4aCompactKWords" |
| }, |
| "passes": [ |
| { |
| "id": "main", |
| "name": "QLinearConv.PointwiseDp4a", |
| "shader": "quant-dp4a-matmul.wgsl.jinja", |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "gemm_params_nchw"], |
| "dispatch": { |
| "x": "ceilDiv(outSpatial, 64)", |
| "y": "ceilDiv(convOutChannels, 4 if dp4aAppleCompact else 8)", |
| "z": "dim(shapes.y, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "matrix_pointwise_nchw", |
| "priority": 30, |
| "when": ["attrs.channels_last == 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "kernelHeight == 1", "kernelWidth == 1", "strideH == 1", "strideW == 1", "padTop == 0 and padBottom == 0", "padLeft == 0 and padRight == 0", "dim(shapes.w, 1) == convInChannels", "outputHeight == inputHeight", "outputWidth == inputWidth", "gemmDispatchOk", "wave32Effective", "sgmatDeviceOk", "sgmatDispatchOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["not (sgmatM >= sgmatTileM and sgmatN >= sgmatTileN / 2 and convK >= sgmatTileK)", "sgmatUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix"], |
| "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 9 } |
| }, |
| "passes": [ |
| { |
| "id": "main", |
| "name": "QLinearConv.ExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "derive": { |
| "aVec4": "aScalar", |
| "matrixHasBias": false, |
| "matrixScalePerRow": "perChannelWeightQuantOk and not channelsLast", |
| "matrixZeroPerRow": "perChannelWeightQuantOk and not channelsLast" |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "gemm_params_nchw_matrix"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, sgmatTileN)", |
| "y": "ceilDiv(sgmatM, sgmatTileM)", |
| "z": "dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "dp4a_im2col_nchw", |
| "priority": 20, |
| "when": ["packedFeature", "attrs.channels_last == 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "outputSpatialOk", "(kernelHeight > 1 or kernelWidth > 1)", "convK >= 64", "convK <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "nchwColsStorageOk", "gemmDispatchOk", "numel(shapes.y) >= 8192"], |
| "derive": { |
| "aPacked": "convK % 4 == 0", |
| "aVec4": "(\"vec4<u32>\" if aUnsigned else \"vec4<i32>\") if aPacked else aScalar", |
| "hasBias": false, |
| "biasPerColumn": "channelsLast", |
| "tileM": "4 if dp4aAppleCompact else 8", |
| "tileN": 64, |
| "ktWords": "dp4aCompactKWords" |
| }, |
| "intermediates": [{ "id": "cols", "dtype": "int32", "shape": "[dim(shapes.x, 0) * convK * outSpatial]" }], |
| "passes": [ |
| { |
| "id": "im2col", |
| "name": "QLinearConv.Im2Col", |
| "shader": "conv-int-im2col-spatial.wgsl.jinja", |
| "derive": { |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft" |
| }, |
| "bindings": ["x_b", "x_zero_point_b", "cols_out_b", "im2col_nchw_params"], |
| "dispatch": { "x": "ceilDiv(outSpatial, tunables.WORKGROUP_SIZE)", "y": "convK", "z": "dim(shapes.x, 0)" } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.Im2ColDp4a", |
| "shader": "quant-dp4a-matmul.wgsl.jinja", |
| "bindings": ["a_weight", "b_cols_nchw", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "gemm_params_nchw"], |
| "dispatch": { |
| "x": "ceilDiv(outSpatial, 64)", |
| "y": "ceilDiv(convOutChannels, 4 if dp4aAppleCompact else 8)", |
| "z": "dim(shapes.y, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "matrix_im2col_nchw", |
| "priority": 30, |
| "when": ["attrs.channels_last == 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "outputSpatialOk", "(kernelHeight > 1 or kernelWidth > 1)", "convK >= 64", "convK <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "nchwColsStorageOk", "gemmDispatchOk", "wave32Effective", "sgmatDeviceOk", "sgmatDispatchOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["not (sgmatM >= sgmatTileM and sgmatN >= sgmatTileN / 2 and convK >= sgmatTileK)", "sgmatUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix"], |
| "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 9 } |
| }, |
| "intermediates": [{ "id": "cols", "dtype": "int32", "shape": "[dim(shapes.x, 0) * convK * outSpatial]" }], |
| "passes": [ |
| { |
| "id": "im2col", |
| "name": "QLinearConv.Im2Col", |
| "shader": "conv-int-im2col-spatial.wgsl.jinja", |
| "derive": { |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft" |
| }, |
| "bindings": ["x_b", "x_zero_point_b", "cols_out_b", "im2col_nchw_params"], |
| "dispatch": { "x": "ceilDiv(outSpatial, tunables.WORKGROUP_SIZE)", "y": "convK", "z": "dim(shapes.x, 0)" } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.ExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "derive": { |
| "aVec4": "aScalar", |
| "matrixHasBias": false, |
| "matrixScalePerRow": "perChannelWeightQuantOk and not channelsLast", |
| "matrixZeroPerRow": "perChannelWeightQuantOk and not channelsLast" |
| }, |
| "bindings": ["a_weight", "b_cols_nchw", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "gemm_params_nchw_matrix"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, sgmatTileN)", |
| "y": "ceilDiv(sgmatM, sgmatTileM)", |
| "z": "dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "dp4a_im2col_nchw_bias", |
| "priority": 25, |
| "when": ["packedFeature", "attrs.channels_last == 0", "biasContractOk", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "outputSpatialOk", "convK <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "nchwColsStorageOk", "gemmDispatchOk"], |
| "derive": { |
| "aPacked": "convK % 4 == 0", |
| "aVec4": "(\"vec4<u32>\" if aUnsigned else \"vec4<i32>\") if aPacked else aScalar", |
| "hasBias": true, |
| "biasPerColumn": "channelsLast", |
| "tileM": "4 if dp4aAppleCompact else 8", |
| "tileN": 64, |
| "ktWords": "dp4aCompactKWords" |
| }, |
| "intermediates": [{ "id": "cols", "dtype": "int32", "shape": "[dim(shapes.x, 0) * convK * outSpatial]" }], |
| "passes": [ |
| { |
| "id": "im2col", |
| "name": "QLinearConv.Im2Col", |
| "shader": "conv-int-im2col-spatial.wgsl.jinja", |
| "derive": { |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft" |
| }, |
| "bindings": ["x_b", "x_zero_point_b", "cols_out_b", "im2col_nchw_params"], |
| "dispatch": { "x": "ceilDiv(outSpatial, tunables.WORKGROUP_SIZE)", "y": "convK", "z": "dim(shapes.x, 0)" } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.Im2ColDp4aBias", |
| "shader": "quant-dp4a-matmul.wgsl.jinja", |
| "bindings": ["a_weight", "b_cols_nchw", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "gemm_params_nchw"], |
| "dispatch": { |
| "x": "ceilDiv(outSpatial, 64)", |
| "y": "ceilDiv(convOutChannels, 4 if dp4aAppleCompact else 8)", |
| "z": "dim(shapes.y, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "matrix_im2col_nchw_bias", |
| "priority": 35, |
| "when": ["attrs.channels_last == 0", "biasContractOk", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "outputSpatialOk", "convK <= min(device.limits.maxComputeWorkgroupsPerDimension, 65535)", "nchwColsStorageOk", "gemmDispatchOk", "wave32Effective", "sgmatDeviceOk", "sgmatDispatchOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["not (sgmatM >= sgmatTileM and sgmatN >= sgmatTileN / 2 and convK >= sgmatTileK)", "sgmatUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix"], |
| "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 10 } |
| }, |
| "intermediates": [{ "id": "cols", "dtype": "int32", "shape": "[dim(shapes.x, 0) * convK * outSpatial]" }], |
| "passes": [ |
| { |
| "id": "im2col", |
| "name": "QLinearConv.Im2Col", |
| "shader": "conv-int-im2col-spatial.wgsl.jinja", |
| "derive": { |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft" |
| }, |
| "bindings": ["x_b", "x_zero_point_b", "cols_out_b", "im2col_nchw_params"], |
| "dispatch": { "x": "ceilDiv(outSpatial, tunables.WORKGROUP_SIZE)", "y": "convK", "z": "dim(shapes.x, 0)" } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.ExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "derive": { |
| "aVec4": "aScalar", |
| "matrixHasBias": true, |
| "matrixScalePerRow": "perChannelWeightQuantOk and not channelsLast", |
| "matrixZeroPerRow": "perChannelWeightQuantOk and not channelsLast" |
| }, |
| "bindings": ["a_weight", "b_cols_nchw", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "gemm_params_nchw_matrix"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, sgmatTileN)", |
| "y": "ceilDiv(sgmatM, sgmatTileM)", |
| "z": "dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "dp4a_pointwise_nhwc", |
| "priority": 20, |
| "when": ["packedFeature", "attrs.channels_last != 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "kernelHeight == 1", "kernelWidth == 1", "strideH == 1", "strideW == 1", "padTop == 0 and padBottom == 0", "padLeft == 0 and padRight == 0", "outputHeight == inputHeight", "outputWidth == inputWidth", "kMajorWeightStorageOk", "gemmDispatchOk", "numel(shapes.y) >= 8192"], |
| "derive": { |
| "aPacked": "convK % 4 == 0", |
| "aVec4": "(\"vec4<u32>\" if aUnsigned else \"vec4<i32>\") if aPacked else aScalar", |
| "hasBias": false, |
| "biasPerColumn": "channelsLast", |
| "tileM": "4 if dp4aAppleCompact else 8", |
| "tileN": 64, |
| "ktWords": "dp4aCompactKWords" |
| }, |
| "intermediates": [{ "id": "w_k_major", "dtype": "int32", "shape": "[convK * convOutChannels]" }], |
| "passes": [ |
| { |
| "id": "weights", |
| "name": "QLinearConv.WeightKMajor", |
| "shader": "quant-weight-k-major.wgsl.jinja", |
| "bindings": ["w_w", "w_zero_point_w", "w_k_major_out", "k_major_params_exact"], |
| "dispatch": { |
| "x": "min(ceilDiv((convK * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((convK * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.NhwcDp4a", |
| "shader": "quant-dp4a-matmul.wgsl.jinja", |
| "bindings": ["a_x", "b_weight_k_major", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "gemm_params_nhwc_pointwise"], |
| "dispatch": { |
| "x": "ceilDiv(convOutChannels, 64)", |
| "y": "ceilDiv(outSpatial, 4 if dp4aAppleCompact else 8)", |
| "z": "dim(shapes.y, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "matrix_pointwise_nhwc", |
| "priority": 30, |
| "when": ["attrs.channels_last != 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "kernelHeight == 1", "kernelWidth == 1", "strideH == 1", "strideW == 1", "padTop == 0 and padBottom == 0", "padLeft == 0 and padRight == 0", "outputHeight == inputHeight", "outputWidth == inputWidth", "kMajorWeightStorageOk", "gemmDispatchOk", "wave32Effective", "sgmatDeviceOk", "sgmatDispatchOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["not (sgmatM >= sgmatTileM and sgmatN >= sgmatTileN / 2 and convK >= sgmatTileK)", "sgmatUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix"], |
| "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 9 } |
| }, |
| "intermediates": [{ "id": "w_k_major", "dtype": "int32", "shape": "[convK * convOutChannels]" }], |
| "passes": [ |
| { |
| "id": "weights", |
| "name": "QLinearConv.WeightKMajor", |
| "shader": "quant-weight-k-major.wgsl.jinja", |
| "bindings": ["w_w", "w_zero_point_w", "w_k_major_out", "k_major_params_exact"], |
| "dispatch": { |
| "x": "min(ceilDiv((convK * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((convK * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.ExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "derive": { |
| "aVec4": "aScalar", |
| "matrixHasBias": false, |
| "matrixScalePerRow": "perChannelWeightQuantOk and not channelsLast", |
| "matrixZeroPerRow": "perChannelWeightQuantOk and not channelsLast" |
| }, |
| "bindings": ["a_x", "b_weight_k_major", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "gemm_params_nhwc_pointwise_matrix"], |
| "dispatch": { "x": "ceilDiv(sgmatN, sgmatTileN)", "y": "ceilDiv(sgmatM, sgmatTileM)" } |
| } |
| ] |
| }, |
| { |
| "id": "dp4a_pointwise_nhwc_bias", |
| "priority": 25, |
| "when": ["packedFeature", "attrs.channels_last != 0", "biasContractOk", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "kernelHeight == 1", "kernelWidth == 1", "strideH == 1", "strideW == 1", "padTop == 0 and padBottom == 0", "padLeft == 0 and padRight == 0", "outputHeight == inputHeight", "outputWidth == inputWidth", "kMajorWeightStorageOk", "gemmDispatchOk"], |
| "derive": { |
| "aPacked": "convK % 4 == 0", |
| "aVec4": "(\"vec4<u32>\" if aUnsigned else \"vec4<i32>\") if aPacked else aScalar", |
| "hasBias": true, |
| "biasPerColumn": "channelsLast", |
| "tileM": "4 if dp4aAppleCompact else 8", |
| "tileN": 64, |
| "ktWords": "dp4aCompactKWords" |
| }, |
| "intermediates": [{ "id": "w_k_major", "dtype": "int32", "shape": "[convK * convOutChannels]" }], |
| "passes": [ |
| { |
| "id": "weights", |
| "name": "QLinearConv.WeightKMajor", |
| "shader": "quant-weight-k-major.wgsl.jinja", |
| "bindings": ["w_w", "w_zero_point_w", "w_k_major_out", "k_major_params_exact"], |
| "dispatch": { |
| "x": "min(ceilDiv((convK * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((convK * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.NhwcDp4aBias", |
| "shader": "quant-dp4a-matmul.wgsl.jinja", |
| "bindings": ["a_x", "b_weight_k_major", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "gemm_params_nhwc_pointwise"], |
| "dispatch": { |
| "x": "ceilDiv(convOutChannels, 64)", |
| "y": "ceilDiv(outSpatial, 4 if dp4aAppleCompact else 8)", |
| "z": "dim(shapes.y, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "matrix_pointwise_nhwc_bias", |
| "priority": 35, |
| "when": ["attrs.channels_last != 0", "biasContractOk", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "kernelHeight == 1", "kernelWidth == 1", "strideH == 1", "strideW == 1", "padTop == 0 and padBottom == 0", "padLeft == 0 and padRight == 0", "outputHeight == inputHeight", "outputWidth == inputWidth", "kMajorWeightStorageOk", "gemmDispatchOk", "wave32Effective", "sgmatDeviceOk", "sgmatDispatchOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["not (sgmatM >= sgmatTileM and sgmatN >= sgmatTileN / 2 and convK >= sgmatTileK)", "sgmatUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix"], |
| "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 10 } |
| }, |
| "intermediates": [{ "id": "w_k_major", "dtype": "int32", "shape": "[convK * convOutChannels]" }], |
| "passes": [ |
| { |
| "id": "weights", |
| "name": "QLinearConv.WeightKMajor", |
| "shader": "quant-weight-k-major.wgsl.jinja", |
| "bindings": ["w_w", "w_zero_point_w", "w_k_major_out", "k_major_params_exact"], |
| "dispatch": { |
| "x": "min(ceilDiv((convK * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((convK * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.ExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "derive": { |
| "aVec4": "aScalar", |
| "matrixHasBias": true, |
| "matrixScalePerRow": "perChannelWeightQuantOk and not channelsLast", |
| "matrixZeroPerRow": "perChannelWeightQuantOk and not channelsLast" |
| }, |
| "bindings": ["a_x", "b_weight_k_major", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "gemm_params_nhwc_pointwise_matrix"], |
| "dispatch": { "x": "ceilDiv(sgmatN, sgmatTileN)", "y": "ceilDiv(sgmatM, sgmatTileM)" } |
| } |
| ] |
| }, |
| { |
| "id": "dp4a_im2col_nhwc", |
| "priority": 20, |
| "when": ["packedFeature", "attrs.channels_last != 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "(kernelHeight > 1 or kernelWidth > 1)", "outputSpatialOk", "nhwcColsStorageOk", "kMajorWeightStorageOk", "gemmDispatchOk", "numel(shapes.y) >= 8192"], |
| "derive": { |
| "aPacked": true, |
| "aVec4": "(\"vec4<u32>\" if aUnsigned else \"vec4<i32>\") if aPacked else aScalar", |
| "hasBias": false, |
| "biasPerColumn": "channelsLast", |
| "tileM": "4 if dp4aAppleCompact else 8", |
| "tileN": 64, |
| "ktWords": "dp4aCompactKWords" |
| }, |
| "intermediates": [ |
| { "id": "w_k_major", "dtype": "int32", "shape": "[convKPadded * convOutChannels]" }, |
| { "id": "cols", "dtype": "int32", "shape": "[dim(shapes.x, 0) * outSpatial * convKPadded]" } |
| ], |
| "passes": [ |
| { |
| "id": "weights", |
| "name": "QLinearConv.WeightKMajor", |
| "shader": "quant-weight-k-major.wgsl.jinja", |
| "bindings": ["w_w", "w_zero_point_w", "w_k_major_out", "k_major_params"], |
| "dispatch": { |
| "x": "min(ceilDiv((convKPadded * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((convKPadded * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| }, |
| { |
| "id": "im2col", |
| "name": "QLinearConv.Im2ColNhwc", |
| "shader": "qlinear-conv-im2col-nhwc.wgsl.jinja", |
| "derive": { |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "im2colWords": "tunables.NHWC_IM2COL_WORDS", |
| "channelsAligned": "convInChannels % 4 == 0", |
| "xVec4": "(\"vec4<u32>\" if tensorDtypes.x == \"uint8\" else \"vec4<i32>\") if convInChannels % 4 == 0 else (\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\")", |
| "colsVec4": "\"vec4<u32>\" if tensorDtypes.x == \"uint8\" else \"vec4<i32>\"" |
| }, |
| "bindings": ["x_im2col", "x_zero_point_w", "cols_out_vec4", "im2col_nhwc_params"], |
| "dispatch": { |
| "x": "min(ceilDiv((ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.NhwcDp4a", |
| "shader": "quant-dp4a-matmul.wgsl.jinja", |
| "bindings": ["a_cols_nhwc", "b_weight_k_major", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "gemm_params_nhwc_im2col"], |
| "dispatch": { |
| "x": "ceilDiv(convOutChannels, 64)", |
| "y": "ceilDiv(outSpatial, 4 if dp4aAppleCompact else 8)", |
| "z": "dim(shapes.y, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "matrix_im2col_nhwc", |
| "priority": 30, |
| "when": ["attrs.channels_last != 0", "not present.B", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "(kernelHeight > 1 or kernelWidth > 1)", "outputSpatialOk", "nhwcColsStorageOk", "kMajorWeightStorageOk", "gemmDispatchOk", "wave32Effective", "sgmatDeviceOk", "sgmatDispatchOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["not (sgmatM >= sgmatTileM and sgmatN >= sgmatTileN / 2 and convK >= sgmatTileK)", "sgmatUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix"], |
| "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 9 } |
| }, |
| "intermediates": [ |
| { "id": "w_k_major", "dtype": "int32", "shape": "[convKPadded * convOutChannels]" }, |
| { "id": "cols", "dtype": "int32", "shape": "[dim(shapes.x, 0) * outSpatial * convKPadded]" } |
| ], |
| "passes": [ |
| { |
| "id": "weights", |
| "name": "QLinearConv.WeightKMajor", |
| "shader": "quant-weight-k-major.wgsl.jinja", |
| "bindings": ["w_w", "w_zero_point_w", "w_k_major_out", "k_major_params"], |
| "dispatch": { |
| "x": "min(ceilDiv((convKPadded * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((convKPadded * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| }, |
| { |
| "id": "im2col", |
| "name": "QLinearConv.Im2ColNhwc", |
| "shader": "qlinear-conv-im2col-nhwc.wgsl.jinja", |
| "derive": { |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "im2colWords": "tunables.NHWC_IM2COL_WORDS", |
| "channelsAligned": "convInChannels % 4 == 0", |
| "xVec4": "(\"vec4<u32>\" if tensorDtypes.x == \"uint8\" else \"vec4<i32>\") if convInChannels % 4 == 0 else (\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\")", |
| "colsVec4": "\"vec4<u32>\" if tensorDtypes.x == \"uint8\" else \"vec4<i32>\"" |
| }, |
| "bindings": ["x_im2col", "x_zero_point_w", "cols_out_vec4", "im2col_nhwc_params"], |
| "dispatch": { |
| "x": "min(ceilDiv((ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.ExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "derive": { |
| "aVec4": "aScalar", |
| "matrixHasBias": false, |
| "matrixScalePerRow": "perChannelWeightQuantOk and not channelsLast", |
| "matrixZeroPerRow": "perChannelWeightQuantOk and not channelsLast" |
| }, |
| "bindings": ["a_cols_nhwc", "b_weight_k_major", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "gemm_params_nhwc_im2col_matrix"], |
| "dispatch": { "x": "ceilDiv(sgmatN, sgmatTileN)", "y": "ceilDiv(sgmatM, sgmatTileM)" } |
| } |
| ] |
| }, |
| { |
| "id": "dp4a_im2col_nhwc_bias", |
| "priority": 25, |
| "when": ["packedFeature", "attrs.channels_last != 0", "biasContractOk", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "(kernelHeight > 1 or kernelWidth > 1)", "outputSpatialOk", "nhwcColsStorageOk", "kMajorWeightStorageOk", "gemmDispatchOk"], |
| "derive": { |
| "aPacked": true, |
| "aVec4": "(\"vec4<u32>\" if aUnsigned else \"vec4<i32>\") if aPacked else aScalar", |
| "hasBias": true, |
| "biasPerColumn": "channelsLast", |
| "tileM": "4 if dp4aAppleCompact else 8", |
| "tileN": 64, |
| "ktWords": "dp4aCompactKWords" |
| }, |
| "intermediates": [ |
| { "id": "w_k_major", "dtype": "int32", "shape": "[convKPadded * convOutChannels]" }, |
| { "id": "cols", "dtype": "int32", "shape": "[dim(shapes.x, 0) * outSpatial * convKPadded]" } |
| ], |
| "passes": [ |
| { |
| "id": "weights", |
| "name": "QLinearConv.WeightKMajor", |
| "shader": "quant-weight-k-major.wgsl.jinja", |
| "bindings": ["w_w", "w_zero_point_w", "w_k_major_out", "k_major_params"], |
| "dispatch": { |
| "x": "min(ceilDiv((convKPadded * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((convKPadded * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| }, |
| { |
| "id": "im2col", |
| "name": "QLinearConv.Im2ColNhwc", |
| "shader": "qlinear-conv-im2col-nhwc.wgsl.jinja", |
| "derive": { |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "im2colWords": "tunables.NHWC_IM2COL_WORDS", |
| "channelsAligned": "convInChannels % 4 == 0", |
| "xVec4": "(\"vec4<u32>\" if tensorDtypes.x == \"uint8\" else \"vec4<i32>\") if convInChannels % 4 == 0 else (\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\")", |
| "colsVec4": "\"vec4<u32>\" if tensorDtypes.x == \"uint8\" else \"vec4<i32>\"" |
| }, |
| "bindings": ["x_im2col", "x_zero_point_w", "cols_out_vec4", "im2col_nhwc_params"], |
| "dispatch": { |
| "x": "min(ceilDiv((ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.NhwcDp4aBias", |
| "shader": "quant-dp4a-matmul.wgsl.jinja", |
| "bindings": ["a_cols_nhwc", "b_weight_k_major", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "gemm_params_nhwc_im2col"], |
| "dispatch": { |
| "x": "ceilDiv(convOutChannels, 64)", |
| "y": "ceilDiv(outSpatial, 4 if dp4aAppleCompact else 8)", |
| "z": "dim(shapes.y, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "matrix_im2col_nhwc_bias", |
| "priority": 35, |
| "when": ["attrs.channels_last != 0", "biasContractOk", "rank4TensorOk", "flexibleWeightQuantOk", "attrs.group == 1", "dim(shapes.w, 1) == convInChannels", "(kernelHeight > 1 or kernelWidth > 1)", "outputSpatialOk", "nhwcColsStorageOk", "kMajorWeightStorageOk", "gemmDispatchOk", "wave32Effective", "sgmatDeviceOk", "sgmatDispatchOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["not (sgmatM >= sgmatTileM and sgmatN >= sgmatTileN / 2 and convK >= sgmatTileK)", "sgmatUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix"], |
| "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 10 } |
| }, |
| "intermediates": [ |
| { "id": "w_k_major", "dtype": "int32", "shape": "[convKPadded * convOutChannels]" }, |
| { "id": "cols", "dtype": "int32", "shape": "[dim(shapes.x, 0) * outSpatial * convKPadded]" } |
| ], |
| "passes": [ |
| { |
| "id": "weights", |
| "name": "QLinearConv.WeightKMajor", |
| "shader": "quant-weight-k-major.wgsl.jinja", |
| "bindings": ["w_w", "w_zero_point_w", "w_k_major_out", "k_major_params"], |
| "dispatch": { |
| "x": "min(ceilDiv((convKPadded * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((convKPadded * convOutChannels), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| }, |
| { |
| "id": "im2col", |
| "name": "QLinearConv.Im2ColNhwc", |
| "shader": "qlinear-conv-im2col-nhwc.wgsl.jinja", |
| "derive": { |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "im2colWords": "tunables.NHWC_IM2COL_WORDS", |
| "channelsAligned": "convInChannels % 4 == 0", |
| "xVec4": "(\"vec4<u32>\" if tensorDtypes.x == \"uint8\" else \"vec4<i32>\") if convInChannels % 4 == 0 else (\"u32\" if tensorDtypes.x == \"uint8\" else \"i32\")", |
| "colsVec4": "\"vec4<u32>\" if tensorDtypes.x == \"uint8\" else \"vec4<i32>\"" |
| }, |
| "bindings": ["x_im2col", "x_zero_point_w", "cols_out_vec4", "im2col_nhwc_params"], |
| "dispatch": { |
| "x": "min(ceilDiv((ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "y": "ceilDiv(ceilDiv((ceilDiv(dim(shapes.x, 0) * outSpatial * (convKPadded / 4), tunables.NHWC_IM2COL_WORDS)), (tunables.WORKGROUP_SIZE)), 65535)", |
| "z": 1 |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.ExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "derive": { |
| "aVec4": "aScalar", |
| "matrixHasBias": true, |
| "matrixScalePerRow": "perChannelWeightQuantOk and not channelsLast", |
| "matrixZeroPerRow": "perChannelWeightQuantOk and not channelsLast" |
| }, |
| "bindings": ["a_cols_nhwc", "b_weight_k_major", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "gemm_params_nhwc_im2col_matrix"], |
| "dispatch": { "x": "ceilDiv(sgmatN, sgmatTileN)", "y": "ceilDiv(sgmatM, sgmatTileM)" } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f16_pointwise_nchw_tensor", |
| "priority": 40, |
| "when": ["precast_pointwise_nchw_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], |
| "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 9 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f16\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": false, |
| "matrixScalePerRow": false, |
| "matrixZeroPerRow": false |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": false, |
| "sampleChannelsLast": false |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_plain"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_params"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f16_pointwise_nchw_tensor", |
| "priority": 28, |
| "when": ["precast_pointwise_nchw_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": ["shader-f16"] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f16\"", |
| "scalar": "\"f16\"", |
| "matrixHasBias": false, |
| "matrixScalePerRow": false, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": false, |
| "sampleChannelsLast": false |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_plain"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f16_pointwise_nchw_row", |
| "priority": 40, |
| "when": ["precast_pointwise_nchw_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], |
| "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 8 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f16\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": false, |
| "matrixScalePerRow": true, |
| "matrixZeroPerRow": true |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": true, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": false, |
| "sampleChannelsLast": false |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_plain"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_params"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f16_pointwise_nchw_row", |
| "priority": 28, |
| "when": ["precast_pointwise_nchw_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": ["shader-f16"] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f16\"", |
| "scalar": "\"f16\"", |
| "matrixHasBias": false, |
| "matrixScalePerRow": true, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": true, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": false, |
| "sampleChannelsLast": false |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_plain"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f16_im2col_nchw_tensor", |
| "priority": 40, |
| "when": ["precast_im2col_nchw_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], |
| "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 9 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f16\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": false, |
| "matrixScalePerRow": false, |
| "matrixZeroPerRow": false |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": true, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": false, |
| "sampleChannelsLast": false |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_params"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f16_im2col_nchw_tensor", |
| "priority": 28, |
| "when": ["precast_im2col_nchw_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": ["shader-f16"] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f16\"", |
| "scalar": "\"f16\"", |
| "matrixHasBias": false, |
| "matrixScalePerRow": false, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": true, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": false, |
| "sampleChannelsLast": false |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f16_im2col_nchw_row", |
| "priority": 40, |
| "when": ["precast_im2col_nchw_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], |
| "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 8 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f16\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": false, |
| "matrixScalePerRow": true, |
| "matrixZeroPerRow": true |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": true, |
| "prepareSpatialB": true, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": false, |
| "sampleChannelsLast": false |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_params"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f16_im2col_nchw_row", |
| "priority": 28, |
| "when": ["precast_im2col_nchw_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": ["shader-f16"] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f16\"", |
| "scalar": "\"f16\"", |
| "matrixHasBias": false, |
| "matrixScalePerRow": true, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": true, |
| "prepareSpatialB": true, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": false, |
| "sampleChannelsLast": false |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f16_im2col_nchw_bias_tensor", |
| "priority": 40, |
| "when": ["precast_im2col_nchw_bias_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], |
| "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 10 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f16\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": true, |
| "matrixScalePerRow": false, |
| "matrixZeroPerRow": false |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": true, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": false, |
| "sampleChannelsLast": false |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_params"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f16_im2col_nchw_bias_tensor", |
| "priority": 28, |
| "when": ["precast_im2col_nchw_bias_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": ["shader-f16"] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f16\"", |
| "scalar": "\"f16\"", |
| "matrixHasBias": true, |
| "matrixScalePerRow": false, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": true, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": false, |
| "sampleChannelsLast": false |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f16_im2col_nchw_bias_row", |
| "priority": 40, |
| "when": ["precast_im2col_nchw_bias_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], |
| "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 9 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f16\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": true, |
| "matrixScalePerRow": true, |
| "matrixZeroPerRow": true |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": true, |
| "prepareSpatialB": true, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": false, |
| "sampleChannelsLast": false |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_params"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f16_im2col_nchw_bias_row", |
| "priority": 28, |
| "when": ["precast_im2col_nchw_bias_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": ["shader-f16"] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f16\"", |
| "scalar": "\"f16\"", |
| "matrixHasBias": true, |
| "matrixScalePerRow": true, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": true, |
| "prepareSpatialB": true, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": false, |
| "sampleChannelsLast": false |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f16_pointwise_nhwc_tensor", |
| "priority": 40, |
| "when": ["precast_pointwise_nhwc_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], |
| "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 9 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f16\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": false, |
| "matrixScalePerRow": false, |
| "matrixZeroPerRow": false |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_nhwc"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f16_pointwise_nhwc_tensor", |
| "priority": 28, |
| "when": ["precast_pointwise_nhwc_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": ["shader-f16"] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f16\"", |
| "scalar": "\"f16\"", |
| "matrixHasBias": false, |
| "matrixScalePerRow": false, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f16_pointwise_nhwc_column", |
| "priority": 40, |
| "when": ["precast_pointwise_nhwc_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], |
| "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 8 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f16\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": false, |
| "matrixScalePerRow": false, |
| "matrixZeroPerRow": false |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "a_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_nhwc"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f16_pointwise_nhwc_column", |
| "priority": 28, |
| "when": ["precast_pointwise_nhwc_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": ["shader-f16"] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f16\"", |
| "scalar": "\"f16\"", |
| "matrixHasBias": false, |
| "matrixScalePerRow": false, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f16_pointwise_nhwc_bias_tensor", |
| "priority": 40, |
| "when": ["precast_pointwise_nhwc_bias_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], |
| "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 10 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f16\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": true, |
| "matrixScalePerRow": false, |
| "matrixZeroPerRow": false |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_nhwc"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f16_pointwise_nhwc_bias_tensor", |
| "priority": 28, |
| "when": ["precast_pointwise_nhwc_bias_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": ["shader-f16"] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f16\"", |
| "scalar": "\"f16\"", |
| "matrixHasBias": true, |
| "matrixScalePerRow": false, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f16_pointwise_nhwc_bias_column", |
| "priority": 40, |
| "when": ["precast_pointwise_nhwc_bias_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], |
| "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 9 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f16\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": true, |
| "matrixScalePerRow": false, |
| "matrixZeroPerRow": false |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "a_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_nhwc"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f16_pointwise_nhwc_bias_column", |
| "priority": 28, |
| "when": ["precast_pointwise_nhwc_bias_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": ["shader-f16"] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f16\"", |
| "scalar": "\"f16\"", |
| "matrixHasBias": true, |
| "matrixScalePerRow": false, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f16_im2col_nhwc_tensor", |
| "priority": 40, |
| "when": ["precast_im2col_nhwc_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], |
| "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 9 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f16\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": false, |
| "matrixScalePerRow": false, |
| "matrixZeroPerRow": false |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": true, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_nhwc"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f16_im2col_nhwc_tensor", |
| "priority": 28, |
| "when": ["precast_im2col_nhwc_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": ["shader-f16"] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f16\"", |
| "scalar": "\"f16\"", |
| "matrixHasBias": false, |
| "matrixScalePerRow": false, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": true, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f16_im2col_nhwc_column", |
| "priority": 40, |
| "when": ["precast_im2col_nhwc_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], |
| "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 8 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f16\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": false, |
| "matrixScalePerRow": false, |
| "matrixZeroPerRow": false |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": true, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "a_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_nhwc"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f16_im2col_nhwc_column", |
| "priority": 28, |
| "when": ["precast_im2col_nhwc_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": ["shader-f16"] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f16\"", |
| "scalar": "\"f16\"", |
| "matrixHasBias": false, |
| "matrixScalePerRow": false, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": true, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f16_im2col_nhwc_bias_tensor", |
| "priority": 40, |
| "when": ["precast_im2col_nhwc_bias_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], |
| "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 10 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f16\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": true, |
| "matrixScalePerRow": false, |
| "matrixZeroPerRow": false |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": true, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_nhwc"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f16_im2col_nhwc_bias_tensor", |
| "priority": 28, |
| "when": ["precast_im2col_nhwc_bias_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": ["shader-f16"] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f16\"", |
| "scalar": "\"f16\"", |
| "matrixHasBias": true, |
| "matrixScalePerRow": false, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": true, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f16_im2col_nhwc_bias_column", |
| "priority": 40, |
| "when": ["precast_im2col_nhwc_bias_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF16BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix", "shader-f16"], |
| "subgroupMatrixConfigs": [{ "componentType": "f16", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 9 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f16\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": true, |
| "matrixScalePerRow": false, |
| "matrixZeroPerRow": false |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": true, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "a_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_nhwc"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f16_im2col_nhwc_bias_column", |
| "priority": 28, |
| "when": ["precast_im2col_nhwc_bias_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF16BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 2 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f16\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": ["shader-f16"] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f16\"", |
| "scalar": "\"f16\"", |
| "matrixHasBias": true, |
| "matrixScalePerRow": false, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float16", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float16", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": true, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f16_a", "portable_f16_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f32_pointwise_nchw_tensor", |
| "priority": 40, |
| "when": ["precast_pointwise_nchw_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix"], |
| "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 9 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f32\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": false, |
| "matrixScalePerRow": false, |
| "matrixZeroPerRow": false |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": false, |
| "sampleChannelsLast": false |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_plain"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_params"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f32_pointwise_nchw_tensor", |
| "priority": 28, |
| "when": ["precast_pointwise_nchw_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": [] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f32\"", |
| "scalar": "\"f32\"", |
| "matrixHasBias": false, |
| "matrixScalePerRow": false, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": false, |
| "sampleChannelsLast": false |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_plain"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f32_pointwise_nchw_row", |
| "priority": 40, |
| "when": ["precast_pointwise_nchw_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix"], |
| "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 8 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f32\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": false, |
| "matrixScalePerRow": true, |
| "matrixZeroPerRow": true |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": true, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": false, |
| "sampleChannelsLast": false |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_plain"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_params"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f32_pointwise_nchw_row", |
| "priority": 28, |
| "when": ["precast_pointwise_nchw_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": [] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f32\"", |
| "scalar": "\"f32\"", |
| "matrixHasBias": false, |
| "matrixScalePerRow": true, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": true, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": false, |
| "sampleChannelsLast": false |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_plain"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f32_im2col_nchw_tensor", |
| "priority": 40, |
| "when": ["precast_im2col_nchw_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix"], |
| "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 9 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f32\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": false, |
| "matrixScalePerRow": false, |
| "matrixZeroPerRow": false |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": true, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": false, |
| "sampleChannelsLast": false |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_params"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f32_im2col_nchw_tensor", |
| "priority": 28, |
| "when": ["precast_im2col_nchw_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": [] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f32\"", |
| "scalar": "\"f32\"", |
| "matrixHasBias": false, |
| "matrixScalePerRow": false, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": true, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": false, |
| "sampleChannelsLast": false |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f32_im2col_nchw_row", |
| "priority": 40, |
| "when": ["precast_im2col_nchw_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix"], |
| "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 8 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f32\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": false, |
| "matrixScalePerRow": true, |
| "matrixZeroPerRow": true |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": true, |
| "prepareSpatialB": true, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": false, |
| "sampleChannelsLast": false |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_params"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f32_im2col_nchw_row", |
| "priority": 28, |
| "when": ["precast_im2col_nchw_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": [] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f32\"", |
| "scalar": "\"f32\"", |
| "matrixHasBias": false, |
| "matrixScalePerRow": true, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": true, |
| "prepareSpatialB": true, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": false, |
| "sampleChannelsLast": false |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f32_im2col_nchw_bias_tensor", |
| "priority": 40, |
| "when": ["precast_im2col_nchw_bias_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix"], |
| "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 10 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f32\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": true, |
| "matrixScalePerRow": false, |
| "matrixZeroPerRow": false |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": true, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": false, |
| "sampleChannelsLast": false |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "a_zero_point_weight", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_params"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f32_im2col_nchw_bias_tensor", |
| "priority": 28, |
| "when": ["precast_im2col_nchw_bias_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": [] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f32\"", |
| "scalar": "\"f32\"", |
| "matrixHasBias": true, |
| "matrixScalePerRow": false, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": true, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": false, |
| "sampleChannelsLast": false |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f32_im2col_nchw_bias_row", |
| "priority": 40, |
| "when": ["precast_im2col_nchw_bias_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix"], |
| "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 9 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f32\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": true, |
| "matrixScalePerRow": true, |
| "matrixZeroPerRow": true |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": true, |
| "prepareSpatialB": true, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": false, |
| "sampleChannelsLast": false |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "b_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_params"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f32_im2col_nchw_bias_row", |
| "priority": 28, |
| "when": ["precast_im2col_nchw_bias_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": [] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f32\"", |
| "scalar": "\"f32\"", |
| "matrixHasBias": true, |
| "matrixScalePerRow": true, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": true, |
| "prepareSpatialB": true, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": false, |
| "sampleChannelsLast": false |
| }, |
| "bindings": ["a_weight", "b_x", "a_zero_point_weight", "b_zero_point_x", "precast_aOut", "precast_bOut", "precast_prepare_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f32_pointwise_nhwc_tensor", |
| "priority": 40, |
| "when": ["precast_pointwise_nhwc_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix"], |
| "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 9 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f32\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": false, |
| "matrixScalePerRow": false, |
| "matrixZeroPerRow": false |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_nhwc"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f32_pointwise_nhwc_tensor", |
| "priority": 28, |
| "when": ["precast_pointwise_nhwc_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": [] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f32\"", |
| "scalar": "\"f32\"", |
| "matrixHasBias": false, |
| "matrixScalePerRow": false, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f32_pointwise_nhwc_column", |
| "priority": 40, |
| "when": ["precast_pointwise_nhwc_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix"], |
| "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 8 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f32\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": false, |
| "matrixScalePerRow": false, |
| "matrixZeroPerRow": false |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "a_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_nhwc"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f32_pointwise_nhwc_column", |
| "priority": 28, |
| "when": ["precast_pointwise_nhwc_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": [] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f32\"", |
| "scalar": "\"f32\"", |
| "matrixHasBias": false, |
| "matrixScalePerRow": false, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f32_pointwise_nhwc_bias_tensor", |
| "priority": 40, |
| "when": ["precast_pointwise_nhwc_bias_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix"], |
| "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 10 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f32\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": true, |
| "matrixScalePerRow": false, |
| "matrixZeroPerRow": false |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_nhwc"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f32_pointwise_nhwc_bias_tensor", |
| "priority": 28, |
| "when": ["precast_pointwise_nhwc_bias_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": [] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f32\"", |
| "scalar": "\"f32\"", |
| "matrixHasBias": true, |
| "matrixScalePerRow": false, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f32_pointwise_nhwc_bias_column", |
| "priority": 40, |
| "when": ["precast_pointwise_nhwc_bias_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix"], |
| "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 9 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f32\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": true, |
| "matrixScalePerRow": false, |
| "matrixZeroPerRow": false |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "a_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_nhwc"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f32_pointwise_nhwc_bias_column", |
| "priority": 28, |
| "when": ["precast_pointwise_nhwc_bias_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": [] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f32\"", |
| "scalar": "\"f32\"", |
| "matrixHasBias": true, |
| "matrixScalePerRow": false, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": false, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_plain"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f32_im2col_nhwc_tensor", |
| "priority": 40, |
| "when": ["precast_im2col_nhwc_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix"], |
| "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 9 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f32\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": false, |
| "matrixScalePerRow": false, |
| "matrixZeroPerRow": false |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": true, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_nhwc"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f32_im2col_nhwc_tensor", |
| "priority": 28, |
| "when": ["precast_im2col_nhwc_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": [] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f32\"", |
| "scalar": "\"f32\"", |
| "matrixHasBias": false, |
| "matrixScalePerRow": false, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": true, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f32_im2col_nhwc_column", |
| "priority": 40, |
| "when": ["precast_im2col_nhwc_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix"], |
| "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 8 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f32\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": false, |
| "matrixScalePerRow": false, |
| "matrixZeroPerRow": false |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": true, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "a_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w", "precast_matrix_nhwc"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f32_im2col_nhwc_column", |
| "priority": 28, |
| "when": ["precast_im2col_nhwc_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": [] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f32\"", |
| "scalar": "\"f32\"", |
| "matrixHasBias": false, |
| "matrixScalePerRow": false, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": true, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f32_im2col_nhwc_bias_tensor", |
| "priority": 40, |
| "when": ["precast_im2col_nhwc_bias_ok", "not perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix"], |
| "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 10 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f32\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": true, |
| "matrixScalePerRow": false, |
| "matrixZeroPerRow": false |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": true, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "a_zero_point_x", "b_zero_point_weight", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_nhwc"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f32_im2col_nhwc_bias_tensor", |
| "priority": 28, |
| "when": ["precast_im2col_nhwc_bias_ok", "not perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": [] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f32\"", |
| "scalar": "\"f32\"", |
| "matrixHasBias": true, |
| "matrixScalePerRow": false, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": true, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "precast_f32_im2col_nhwc_bias_column", |
| "priority": 40, |
| "when": ["precast_im2col_nhwc_bias_ok", "perChannelWeightQuantOk", "wave32Effective", "precastResourcesOk", "precastDispatchOk", "precastF32BuffersOk", "device.wgslLanguageFeatures.has(\"subgroup_id\")"], |
| "demoteWhen": ["precastAmortizedInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS"], |
| "requires": { |
| "features": ["subgroups", "chromium-experimental-subgroup-matrix"], |
| "subgroupMatrixConfigs": [{ "componentType": "f32", "resultComponentType": "f32", "M": 8, "N": 8, "K": 8 }], |
| "limits": { "maxStorageBuffersPerShaderStage": 9 } |
| }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastOperands": true, |
| "precastScalar": "\"f32\"", |
| "sgmatTileM": "precastTileM", |
| "sgmatTileN": "precastTileN", |
| "sgmatTileK": "precastTileK", |
| "sgmatRowGroups": "precastSubgroups", |
| "sgmatSubgroups": "precastSubgroups", |
| "sgmatWorkgroup": "precastWorkgroup", |
| "matrixHasBias": true, |
| "matrixScalePerRow": false, |
| "matrixZeroPerRow": false |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": true, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactMatrix", |
| "shader": "quant-exact-matrix.wgsl.jinja", |
| "bindings": ["precast_a", "precast_b", "a_zero_point_x", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w", "precast_matrix_nhwc"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, precastTileN)", |
| "y": "ceilDiv(sgmatM, precastTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| }, |
| { |
| "id": "portable_f32_im2col_nhwc_bias_column", |
| "priority": 28, |
| "when": ["precast_im2col_nhwc_bias_ok", "perChannelWeightQuantOk", "precastDispatchOk", "precastF32BuffersOk", "portableGeometryOk", "(portableTileM + portableTileN) * portableTileK * 4 <= device.limits.maxComputeWorkgroupStorageSize", "portableLoadsOk"], |
| "demoteWhen": ["precastUsefulInvocations < tunables.EXACT_MATRIX_MIN_INVOCATIONS and not (has(device.adapterInfo, \"vendor\") and device.adapterInfo.vendor == \"apple\")", "\"f32\" == \"f32\" and sgmatM < portableTileN", "dp4aAppleCompact and packedFeature"], |
| "requires": { "features": [] }, |
| "derive": { |
| "aVec4": "aScalar", |
| "precastScalar": "\"f32\"", |
| "scalar": "\"f32\"", |
| "matrixHasBias": true, |
| "matrixScalePerRow": false, |
| "M": "sgmatM", |
| "N": "sgmatN", |
| "K": "preparedK", |
| "kTile": "portableTileK", |
| "workgroupCols": "portableWorkgroupCols", |
| "workgroupRows": "portableWorkgroupRows", |
| "blockRows": "portableTileM", |
| "blockCols": "portableTileN", |
| "integerPartialK": "portablePartialK", |
| "hasBias": false, |
| "exactBatchB": "not channelsLast and dim(shapes.x, 0) > 1" |
| }, |
| "intermediates": [ |
| { "id": "aFloat", "dtype": "float32", "shape": "[preparedElements]" }, |
| { "id": "bFloat", "dtype": "float32", "shape": "[preparedBElements]" } |
| ], |
| "passes": [ |
| { |
| "id": "prepare", |
| "name": "QLinearConv.PrepareExact", |
| "shader": "quant-exact-prepare.wgsl.jinja", |
| "derive": { |
| "prepareZeroPerRow": false, |
| "prepareSpatialB": false, |
| "spatialRankSpec": 2, |
| "kernelHSpec": "kernelHeight", |
| "kernelWSpec": "kernelWidth", |
| "strideHSpec": "strideH", |
| "strideWSpec": "strideW", |
| "dilationHSpec": "dilationH", |
| "dilationWSpec": "dilationW", |
| "padTopSpec": "effectivePadTop", |
| "padLeftSpec": "effectivePadLeft", |
| "prepareSpatialA": true, |
| "prepareTransposeB": true, |
| "sampleChannelsLast": true |
| }, |
| "bindings": ["a_x", "precast_weight_oihw", "a_zero_point_x", "b_zero_point_weight", "precast_aOut", "precast_bOut", "precast_prepare_nhwc_spatial"], |
| "dispatch": { |
| "x": "min(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))", |
| "y": "ceilDiv(prepareGroups, min(device.limits.maxComputeWorkgroupsPerDimension, 65535))" |
| } |
| }, |
| { |
| "id": "main", |
| "name": "QLinearConv.PreparedExactPortable", |
| "shader": "quant-exact-portable.wgsl.jinja", |
| "bindings": ["portable_f32_a", "portable_f32_b", "a_scale_x", "b_scale_w", "y_scale", "y_zero_point_w", "bias", "y_w"], |
| "dispatch": { |
| "x": "ceilDiv(sgmatN, portableTileN)", |
| "y": "ceilDiv(sgmatM, portableTileM)", |
| "z": "1 if channelsLast else dim(shapes.x, 0)" |
| } |
| } |
| ] |
| } |
| ] |
| } |
|
|
