File size: 6,119 Bytes
2abfed0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
"""Weight fake-quantization layers for registered low-bit experiments."""

from __future__ import annotations

import tensorflow as tf

from imagenet_research.models import (
    BiFPNLayer,
    DCNv4Reference,
    SpatialTransformerBlock,
    _fake_quantize_kernel_per_output_channel,
)


@tf.keras.utils.register_keras_serializable(package="imagenet_research")
class FakeQuantConv2D(tf.keras.layers.Conv2D):
    """Conv2D with per-output-channel fake-quantized kernels."""

    def __init__(self, num_bits: int = 4, **kwargs):
        if num_bits < 2 or num_bits > 8:
            raise ValueError("num_bits must be between 2 and 8")
        self.num_bits = int(num_bits)
        super().__init__(**kwargs)

    def convolution_op(self, inputs, kernel):
        quantized_kernel = _fake_quantize_kernel_per_output_channel(
            kernel, self.num_bits
        )
        return super().convolution_op(inputs, tf.cast(quantized_kernel, kernel.dtype))

    def get_config(self):
        config = super().get_config()
        config["num_bits"] = self.num_bits
        return config


@tf.keras.utils.register_keras_serializable(package="imagenet_research")
class FakeQuantDepthwiseConv2D(tf.keras.layers.DepthwiseConv2D):
    """DepthwiseConv2D with per-channel fake-quantized kernels."""

    def __init__(self, num_bits: int = 4, **kwargs):
        if num_bits < 2 or num_bits > 8:
            raise ValueError("num_bits must be between 2 and 8")
        self.num_bits = int(num_bits)
        super().__init__(**kwargs)

    def call(self, inputs):
        kernel32 = tf.cast(self.kernel, tf.float32)
        input_channels = self._get_input_channel(inputs.shape)
        output_channels = input_channels * self.depth_multiplier
        flattened = tf.reshape(
            kernel32, [self.kernel_size[0], self.kernel_size[1], output_channels]
        )
        quantized = _fake_quantize_kernel_per_output_channel(flattened, self.num_bits)
        quantized = tf.reshape(quantized, tf.shape(kernel32))
        quantized = tf.cast(quantized, self.compute_dtype)
        inputs = tf.cast(inputs, self.compute_dtype)

        outputs = tf.keras.ops.depthwise_conv(
            inputs,
            quantized,
            strides=self.strides,
            padding=self.padding,
            dilation_rate=self.dilation_rate,
            data_format=self.data_format,
        )
        if self.use_bias:
            bias = tf.cast(self.bias, self.compute_dtype)
            if self.data_format == "channels_last":
                bias_shape = (1,) * (self.rank + 1) + (output_channels,)
            else:
                bias_shape = (1, output_channels) + (1,) * self.rank
            outputs = tf.keras.ops.add(outputs, tf.keras.ops.reshape(bias, bias_shape))
        if self.activation is not None:
            outputs = self.activation(outputs)
        return outputs

    def get_config(self):
        config = super().get_config()
        config["num_bits"] = self.num_bits
        return config


@tf.keras.utils.register_keras_serializable(package="imagenet_research")
class FakeQuantDense(tf.keras.layers.Dense):
    """Dense with per-output-channel fake-quantized kernels."""

    def __init__(self, num_bits: int = 4, **kwargs):
        if num_bits < 2 or num_bits > 8:
            raise ValueError("num_bits must be between 2 and 8")
        self.num_bits = int(num_bits)
        super().__init__(**kwargs)

    def call(self, inputs, training=None):
        kernel = _fake_quantize_kernel_per_output_channel(self.kernel, self.num_bits)
        kernel = tf.cast(kernel, self.compute_dtype)
        inputs = tf.cast(inputs, self.compute_dtype)
        outputs = tf.keras.ops.matmul(inputs, kernel)
        if self.bias is not None:
            outputs = tf.keras.ops.add(outputs, tf.cast(self.bias, self.compute_dtype))
        if self.activation is not None:
            outputs = self.activation(outputs)
        return outputs

    def get_config(self):
        config = super().get_config()
        config["num_bits"] = self.num_bits
        return config


def apply_weight_fake_quant(model: tf.keras.Model, num_bits: int) -> tf.keras.Model:
    """Clone a supported classifier with trainable, per-output-channel Q/DQ.

    The fake-quant ops use TensorFlow's straight-through estimator, so the
    returned model can be evaluated as PTQ simulation or fine-tuned as QAT.
    Hybrid and BiFPN custom layers quantize their internal convolutional and
    dense projection kernels, including the MHSA projections. Normalization
    parameters, biases, softmax, and activations remain float32. DCNv4's custom
    layer quantizes only its value/output projections; its offset and
    aggregation-weight predictor remains float32.
    """
    if num_bits < 2 or num_bits > 8:
        raise ValueError("num_bits must be between 2 and 8")

    def clone_layer(layer):
        config = layer.get_config()
        if isinstance(layer, DCNv4Reference):
            config["main_weight_bits"] = num_bits
            return DCNv4Reference.from_config(config)
        if isinstance(layer, SpatialTransformerBlock):
            config["weight_fake_quant_bits"] = num_bits
            return SpatialTransformerBlock.from_config(config)
        if isinstance(layer, BiFPNLayer):
            config["weight_fake_quant_bits"] = num_bits
            return BiFPNLayer.from_config(config)
        if isinstance(layer, tf.keras.layers.DepthwiseConv2D):
            return FakeQuantDepthwiseConv2D(num_bits=num_bits, **config)
        if isinstance(layer, tf.keras.layers.Conv2D):
            return FakeQuantConv2D(num_bits=num_bits, **config)
        if isinstance(layer, tf.keras.layers.Dense):
            return FakeQuantDense(num_bits=num_bits, **config)
        return layer.__class__.from_config(config)

    clone = tf.keras.models.clone_model(
        model, clone_function=clone_layer, recursive=True
    )
    for source_layer in model.layers:
        target_layer = clone.get_layer(source_layer.name)
        if source_layer.weights:
            target_layer.set_weights(source_layer.get_weights())
    return clone