Image Classification
Keras
LiteRT
LiteRT
computer-vision
efficientnet
quantization
mixed-precision
sub-4-bit
tensorflow
Eval Results (legacy)
Instructions to use meatloafer/efficientnet-b0-sub4bit-mixed-precision with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Keras
How to use meatloafer/efficientnet-b0-sub4bit-mixed-precision with Keras:
# !pip install -U keras tensorflow huggingface_hub # Keras needs TensorFlow installed to read "hf://" paths, so the tensorflow backend is selected here; # "jax" and "torch" also work for computation once TensorFlow is installed. import os os.environ["KERAS_BACKEND"] = "tensorflow" import keras model = keras.saving.load_model("hf://meatloafer/efficientnet-b0-sub4bit-mixed-precision") - LiteRT
How to use meatloafer/efficientnet-b0-sub4bit-mixed-precision with LiteRT:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
Upload verified model weights, LiteRT FlatBuffers, DP profiles, code, and model card
2abfed0 verified Download code/models.py from meatloafer/efficientnet-b0-sub4bit-mixed-precision: direct link, hf CLI and curl.
- Browser
- Download file 37.5 kB
-
https://huggingface.co/meatloafer/efficientnet-b0-sub4bit-mixed-precision/resolve/main/code/models.py
- Command line
-
hf download hf://meatloafer/efficientnet-b0-sub4bit-mixed-precision/code/models.py
-
curl -L -o models.py https://huggingface.co/meatloafer/efficientnet-b0-sub4bit-mixed-precision/resolve/main/code/models.py
37.5 kB
| """TensorFlow/Keras EfficientNet, BiFPN, and CNN-attention classifiers. | |
| All convolutional blocks are implemented here rather than loaded from a | |
| pretrained application model. The default activation is ReLU6 for the planned | |
| integer-quantization comparisons. | |
| """ | |
| from __future__ import annotations | |
| from dataclasses import dataclass | |
| import math | |
| import tensorflow as tf | |
| class Variant: | |
| width: float | |
| depth: float | |
| image_size: int | |
| dropout: float | |
| VARIANTS = { | |
| "b0": Variant(width=1.0, depth=1.0, image_size=224, dropout=0.2), | |
| "b1": Variant(width=1.0, depth=1.1, image_size=240, dropout=0.2), | |
| "b2": Variant(width=1.1, depth=1.2, image_size=260, dropout=0.3), | |
| } | |
| STAGE_SPECS = ( | |
| # kernel, expansion, channels, repeats, first stride | |
| (3, 1, 16, 1, 1), | |
| (3, 6, 24, 2, 2), | |
| (5, 6, 40, 2, 2), | |
| (3, 6, 80, 3, 2), | |
| (5, 6, 112, 3, 1), | |
| (5, 6, 192, 4, 2), | |
| (3, 6, 320, 1, 1), | |
| ) | |
| def round_filters(filters: int, width: float, divisor: int = 8) -> int: | |
| scaled = filters * width | |
| rounded = max(divisor, int(scaled + divisor / 2) // divisor * divisor) | |
| if rounded < 0.9 * scaled: | |
| rounded += divisor | |
| return int(rounded) | |
| def round_repeats(repeats: int, depth: float) -> int: | |
| return int(math.ceil(repeats * depth)) | |
| def _conv_bn_relu6(x, filters, kernel, stride, name, activation=True): | |
| x = tf.keras.layers.Conv2D( | |
| filters, | |
| kernel, | |
| strides=stride, | |
| padding="same", | |
| use_bias=False, | |
| kernel_initializer="he_normal", | |
| name=f"{name}_conv", | |
| )(x) | |
| x = tf.keras.layers.BatchNormalization( | |
| momentum=0.99, epsilon=1e-3, name=f"{name}_bn" | |
| )(x) | |
| if activation: | |
| x = tf.keras.layers.ReLU(max_value=6.0, name=f"{name}_relu6")(x) | |
| return x | |
| def _fake_quantize_kernel_per_output_channel(kernel, num_bits: int): | |
| """Apply symmetric, narrow-range fake quantization to a weight tensor.""" | |
| kernel32 = tf.cast(kernel, tf.float32) | |
| reduction_axes = tuple(range(kernel32.shape.rank - 1)) | |
| bound = tf.reduce_max(tf.abs(kernel32), axis=reduction_axes) | |
| bound = tf.stop_gradient(tf.maximum(bound, 1e-8)) | |
| return tf.quantization.fake_quant_with_min_max_vars_per_channel( | |
| kernel32, | |
| min=-bound, | |
| max=bound, | |
| num_bits=num_bits, | |
| narrow_range=True, | |
| ) | |
| def _conv2d_with_fake_quant(layer, inputs, num_bits: int | None): | |
| compute_dtype = tf.as_dtype(layer.compute_dtype) | |
| inputs = tf.cast(inputs, compute_dtype) | |
| if num_bits is None: | |
| return layer(inputs) | |
| if not layer.built: | |
| layer.build(inputs.shape) | |
| kernel = _fake_quantize_kernel_per_output_channel(layer.kernel, num_bits) | |
| kernel = tf.cast(kernel, compute_dtype) | |
| output = tf.nn.conv2d( | |
| inputs, | |
| kernel, | |
| strides=[1, *layer.strides, 1], | |
| padding=layer.padding.upper(), | |
| data_format="NHWC", | |
| dilations=[1, *layer.dilation_rate, 1], | |
| ) | |
| if layer.use_bias: | |
| output = tf.nn.bias_add( | |
| output, tf.cast(layer.bias, compute_dtype), data_format="NHWC" | |
| ) | |
| if layer.activation is not None: | |
| output = layer.activation(output) | |
| return output | |
| def _depthwise_conv_with_fake_quant(layer, inputs, num_bits: int | None): | |
| if num_bits is None: | |
| return layer(inputs) | |
| if not layer.built: | |
| layer.build(inputs.shape) | |
| kernel32 = tf.cast(layer.kernel, tf.float32) | |
| input_channels = int(kernel32.shape[-2]) | |
| output_channels = input_channels * layer.depth_multiplier | |
| flattened = tf.reshape( | |
| kernel32, [layer.kernel_size[0], layer.kernel_size[1], output_channels] | |
| ) | |
| quantized = _fake_quantize_kernel_per_output_channel(flattened, num_bits) | |
| kernel = tf.cast(tf.reshape(quantized, tf.shape(kernel32)), layer.compute_dtype) | |
| inputs = tf.cast(inputs, layer.compute_dtype) | |
| output = tf.keras.ops.depthwise_conv( | |
| inputs, | |
| kernel, | |
| strides=layer.strides, | |
| padding=layer.padding, | |
| dilation_rate=layer.dilation_rate, | |
| data_format=layer.data_format, | |
| ) | |
| if layer.use_bias: | |
| output = tf.nn.bias_add( | |
| output, tf.cast(layer.bias, layer.compute_dtype), data_format="NHWC" | |
| ) | |
| if layer.activation is not None: | |
| output = layer.activation(output) | |
| return output | |
| def _separable_conv_with_fake_quant(layer, inputs, num_bits: int | None): | |
| if num_bits is None: | |
| return layer(inputs) | |
| if not layer.built: | |
| layer.build(inputs.shape) | |
| depthwise32 = tf.cast(layer.depthwise_kernel, tf.float32) | |
| output_channels = int(depthwise32.shape[-2]) * layer.depth_multiplier | |
| flattened = tf.reshape( | |
| depthwise32, | |
| [layer.kernel_size[0], layer.kernel_size[1], output_channels], | |
| ) | |
| depthwise = _fake_quantize_kernel_per_output_channel(flattened, num_bits) | |
| depthwise = tf.cast(tf.reshape(depthwise, tf.shape(depthwise32)), layer.compute_dtype) | |
| pointwise = _fake_quantize_kernel_per_output_channel( | |
| layer.pointwise_kernel, num_bits | |
| ) | |
| pointwise = tf.cast(pointwise, layer.compute_dtype) | |
| inputs = tf.cast(inputs, layer.compute_dtype) | |
| output = tf.keras.ops.separable_conv( | |
| inputs, | |
| depthwise, | |
| pointwise, | |
| strides=layer.strides, | |
| padding=layer.padding, | |
| data_format=layer.data_format, | |
| dilation_rate=layer.dilation_rate, | |
| ) | |
| if layer.use_bias: | |
| output = tf.nn.bias_add( | |
| output, tf.cast(layer.bias, layer.compute_dtype), data_format="NHWC" | |
| ) | |
| if layer.activation is not None: | |
| output = layer.activation(output) | |
| return output | |
| def _dense_with_fake_quant(layer, inputs, num_bits: int | None): | |
| if num_bits is None: | |
| return layer(inputs) | |
| if not layer.built: | |
| layer.build(inputs.shape) | |
| kernel = _fake_quantize_kernel_per_output_channel(layer.kernel, num_bits) | |
| output = tf.keras.ops.matmul( | |
| tf.cast(inputs, layer.compute_dtype), tf.cast(kernel, layer.compute_dtype) | |
| ) | |
| if layer.bias is not None: | |
| output = tf.keras.ops.add(output, tf.cast(layer.bias, layer.compute_dtype)) | |
| if layer.activation is not None: | |
| output = layer.activation(output) | |
| return output | |
| def _einsum_dense_with_fake_quant( | |
| layer, inputs, num_bits: int, *, flatten_output_dimensions: bool = False | |
| ): | |
| if not layer.built: | |
| layer.build(inputs.shape) | |
| kernel32 = tf.cast(layer.kernel, tf.float32) | |
| if flatten_output_dimensions: | |
| original_shape = tf.shape(kernel32) | |
| flat_shape = tf.stack([original_shape[0], -1]) | |
| flattened = tf.reshape(kernel32, flat_shape) | |
| quantized = _fake_quantize_kernel_per_output_channel(flattened, num_bits) | |
| kernel = tf.reshape(quantized, original_shape) | |
| else: | |
| kernel = _fake_quantize_kernel_per_output_channel(kernel32, num_bits) | |
| output = tf.keras.ops.einsum( | |
| layer.equation, | |
| tf.cast(inputs, layer.compute_dtype), | |
| tf.cast(kernel, layer.compute_dtype), | |
| ) | |
| if layer.bias is not None: | |
| output = tf.keras.ops.add(output, tf.cast(layer.bias, layer.compute_dtype)) | |
| if layer.activation is not None: | |
| output = layer.activation(output) | |
| return output | |
| def _kernel_parameter_count(layer) -> int: | |
| count = 0 | |
| for name in ("kernel", "depthwise_kernel", "pointwise_kernel"): | |
| kernel = getattr(layer, name, None) | |
| if kernel is not None: | |
| count += int(tf.TensorShape(kernel.shape).num_elements() or 0) | |
| return count | |
| class DCNv4Reference(tf.keras.layers.Layer): | |
| """Differentiable TensorFlow reference for grouped DCNv4 aggregation. | |
| This implements the published operator's value projection, learned offsets | |
| and unnormalized aggregation weights, bilinear sampling, and output | |
| projection. It is a portable reference implementation; it does not include | |
| OpenGVLab's fused CUDA kernel or its memory-access optimizations. | |
| """ | |
| def __init__( | |
| self, | |
| channels: int, | |
| groups: int = 8, | |
| kernel_size: int = 3, | |
| offset_scale: float = 1.0, | |
| main_weight_bits: int | None = None, | |
| **kwargs, | |
| ): | |
| super().__init__(**kwargs) | |
| if channels % groups: | |
| raise ValueError(f"channels={channels} must be divisible by groups={groups}") | |
| if kernel_size < 1 or kernel_size % 2 == 0: | |
| raise ValueError("kernel_size must be a positive odd number") | |
| if main_weight_bits not in (None, 4, 8): | |
| raise ValueError("main_weight_bits must be None, 4, or 8") | |
| self.channels = channels | |
| self.groups = groups | |
| self.kernel_size = kernel_size | |
| self.offset_scale = float(offset_scale) | |
| self.main_weight_bits = main_weight_bits | |
| self.points = kernel_size * kernel_size | |
| self.value_proj = tf.keras.layers.Conv2D( | |
| channels, 1, use_bias=True, kernel_initializer="glorot_uniform", name="value_proj" | |
| ) | |
| self.offset_mask_projection = tf.keras.layers.Conv2D( | |
| groups * self.points * 3, | |
| 1, | |
| use_bias=True, | |
| kernel_initializer="zeros", | |
| bias_initializer="zeros", | |
| dtype="float32", | |
| name="offset_mask_projection", | |
| ) | |
| self.output_proj = tf.keras.layers.Conv2D( | |
| channels, 1, use_bias=True, kernel_initializer="glorot_uniform", name="output_proj" | |
| ) | |
| def build(self, input_shape): | |
| if input_shape[-1] is not None and int(input_shape[-1]) != self.channels: | |
| raise ValueError( | |
| f"DCNv4 channels={self.channels} does not match input channels={input_shape[-1]}" | |
| ) | |
| self.value_proj.build(input_shape) | |
| self.offset_mask_projection.build(input_shape) | |
| self.output_proj.build(input_shape) | |
| super().build(input_shape) | |
| def _aggregate(self, values, offsets, aggregation_weights): | |
| shape = tf.shape(values) | |
| batch, height, width = shape[0], shape[1], shape[2] | |
| channels_per_group = self.channels // self.groups | |
| padding = self.kernel_size // 2 | |
| # OpenGVLab enumerates sampling points with width outermost; keep this | |
| # order aligned with the predictor's per-point offset and weight slots. | |
| kernel_offsets = tf.constant( | |
| [ | |
| (row - padding, column - padding) | |
| for column in range(self.kernel_size) | |
| for row in range(self.kernel_size) | |
| ], | |
| dtype=tf.float32, | |
| ) | |
| grid_y = tf.cast(tf.range(height), tf.float32)[None, :, None, None, None] | |
| grid_x = tf.cast(tf.range(width), tf.float32)[None, None, :, None, None] | |
| point_y = kernel_offsets[:, 0][None, None, None, None, :] | |
| point_x = kernel_offsets[:, 1][None, None, None, None, :] | |
| # The official DCNv4 predictor stores each displacement as (x, y), | |
| # then scales both the regular kernel lattice and learned displacement. | |
| sample_y = ( | |
| grid_y + point_y * self.offset_scale + offsets[..., 1] * self.offset_scale | |
| ) | |
| sample_x = ( | |
| grid_x + point_x * self.offset_scale + offsets[..., 0] * self.offset_scale | |
| ) | |
| y0_float = tf.floor(sample_y) | |
| x0_float = tf.floor(sample_x) | |
| y0 = tf.cast(y0_float, tf.int32) | |
| x0 = tf.cast(x0_float, tf.int32) | |
| y1 = y0 + 1 | |
| x1 = x0 + 1 | |
| fraction_y = sample_y - y0_float | |
| fraction_x = sample_x - x0_float | |
| grouped_values = tf.reshape( | |
| values, | |
| [batch, height, width, self.groups, channels_per_group], | |
| ) | |
| grouped_values = tf.transpose(grouped_values, [0, 3, 1, 2, 4]) | |
| grouped_values = tf.reshape( | |
| grouped_values, | |
| [batch * self.groups, height * width, channels_per_group], | |
| ) | |
| def gather_neighbour(y_index, x_index): | |
| valid = ( | |
| (y_index >= 0) | |
| & (y_index < height) | |
| & (x_index >= 0) | |
| & (x_index < width) | |
| ) | |
| flat_index = tf.clip_by_value(y_index, 0, height - 1) * width | |
| flat_index += tf.clip_by_value(x_index, 0, width - 1) | |
| flat_index = tf.transpose(flat_index, [0, 3, 1, 2, 4]) | |
| valid = tf.transpose(valid, [0, 3, 1, 2, 4]) | |
| flat_index = tf.reshape(flat_index, [batch * self.groups, height * width * self.points]) | |
| valid = tf.reshape(valid, [batch * self.groups, height * width * self.points]) | |
| samples = tf.gather( | |
| grouped_values, flat_index, axis=1, batch_dims=1 | |
| ) | |
| samples = tf.reshape( | |
| samples, | |
| [batch, self.groups, height * width, self.points, channels_per_group], | |
| ) | |
| valid = tf.reshape( | |
| valid, [batch, self.groups, height * width, self.points, 1] | |
| ) | |
| return samples * tf.cast(valid, samples.dtype) | |
| fraction_y = tf.transpose(fraction_y, [0, 3, 1, 2, 4]) | |
| fraction_x = tf.transpose(fraction_x, [0, 3, 1, 2, 4]) | |
| fraction_y = tf.reshape(fraction_y, [batch, self.groups, height * width, self.points]) | |
| fraction_x = tf.reshape(fraction_x, [batch, self.groups, height * width, self.points]) | |
| weights = tf.transpose(aggregation_weights, [0, 3, 1, 2, 4]) | |
| weights = tf.reshape(weights, [batch, self.groups, height * width, self.points]) | |
| top_left = gather_neighbour(y0, x0) | |
| top_right = gather_neighbour(y0, x1) | |
| bottom_left = gather_neighbour(y1, x0) | |
| bottom_right = gather_neighbour(y1, x1) | |
| sampled = ( | |
| top_left * ((1.0 - fraction_y) * (1.0 - fraction_x))[..., None] | |
| + top_right * ((1.0 - fraction_y) * fraction_x)[..., None] | |
| + bottom_left * (fraction_y * (1.0 - fraction_x))[..., None] | |
| + bottom_right * (fraction_y * fraction_x)[..., None] | |
| ) | |
| aggregated = tf.reduce_sum(sampled * weights[..., None], axis=3) | |
| aggregated = tf.transpose(aggregated, [0, 2, 1, 3]) | |
| return tf.reshape(aggregated, [batch, height, width, self.channels]) | |
| def call(self, inputs): | |
| original_dtype = inputs.dtype | |
| inputs32 = tf.cast(inputs, tf.float32) | |
| values = _conv2d_with_fake_quant( | |
| self.value_proj, inputs32, self.main_weight_bits | |
| ) | |
| offset_mask = self.offset_mask_projection(inputs32) | |
| shape = tf.shape(offset_mask) | |
| offset_mask = tf.reshape( | |
| offset_mask, | |
| [shape[0], shape[1], shape[2], self.groups, self.points * 3], | |
| ) | |
| # DCNv4 packs each group's 2*K x/y offsets first, followed by K | |
| # aggregation weights. Keep this ordering aligned with the official | |
| # kernel's group stride and its p_offset_ptr + 2*K weight offset. | |
| offsets = tf.reshape( | |
| offset_mask[..., : self.points * 2], | |
| [shape[0], shape[1], shape[2], self.groups, self.points, 2], | |
| ) | |
| aggregation_weights = offset_mask[..., self.points * 2 :] | |
| output = self._aggregate( | |
| tf.cast(values, tf.float32), offsets, aggregation_weights | |
| ) | |
| output = _conv2d_with_fake_quant( | |
| self.output_proj, output, self.main_weight_bits | |
| ) | |
| return tf.cast(output, original_dtype) | |
| def compute_output_shape(self, input_shape): | |
| return input_shape | |
| def get_config(self): | |
| config = super().get_config() | |
| config.update( | |
| { | |
| "channels": self.channels, | |
| "groups": self.groups, | |
| "kernel_size": self.kernel_size, | |
| "offset_scale": self.offset_scale, | |
| "main_weight_bits": self.main_weight_bits, | |
| } | |
| ) | |
| return config | |
| def _mbconv( | |
| x, | |
| filters_in, | |
| filters_out, | |
| kernel, | |
| stride, | |
| expansion, | |
| se_ratio, | |
| drop_rate, | |
| name, | |
| dcnv4_spatial: bool = False, | |
| dcnv4_main_weight_bits: int | None = None, | |
| ): | |
| residual = x | |
| expanded_filters = filters_in * expansion | |
| if expansion != 1: | |
| x = _conv_bn_relu6( | |
| x, expanded_filters, 1, 1, f"{name}_expand", activation=True | |
| ) | |
| if dcnv4_spatial: | |
| if stride != 1 or kernel != 3: | |
| raise ValueError("The DCNv4 reference path currently supports stride-1 3x3 blocks") | |
| x = DCNv4Reference( | |
| channels=expanded_filters, | |
| groups=8, | |
| kernel_size=3, | |
| main_weight_bits=dcnv4_main_weight_bits, | |
| name=f"{name}_dcnv4", | |
| )(x) | |
| else: | |
| x = tf.keras.layers.DepthwiseConv2D( | |
| kernel, | |
| strides=stride, | |
| padding="same", | |
| use_bias=False, | |
| depthwise_initializer="he_normal", | |
| name=f"{name}_depthwise", | |
| )(x) | |
| x = tf.keras.layers.BatchNormalization( | |
| momentum=0.99, epsilon=1e-3, name=f"{name}_depthwise_bn" | |
| )(x) | |
| x = tf.keras.layers.ReLU(max_value=6.0, name=f"{name}_depthwise_relu6")(x) | |
| # The SE gate acts on the expanded tensor, but its squeeze width is based | |
| # on the MBConv input channels. | |
| squeeze_filters = max(1, int(filters_in * se_ratio)) | |
| se = tf.keras.layers.GlobalAveragePooling2D(keepdims=True, name=f"{name}_se_pool")(x) | |
| se = tf.keras.layers.Conv2D( | |
| squeeze_filters, | |
| 1, | |
| activation="relu", | |
| kernel_initializer="he_normal", | |
| name=f"{name}_se_reduce", | |
| )(se) | |
| se = tf.keras.layers.Conv2D( | |
| expanded_filters, | |
| 1, | |
| activation="sigmoid", | |
| kernel_initializer="glorot_uniform", | |
| name=f"{name}_se_expand", | |
| )(se) | |
| x = tf.keras.layers.Multiply(name=f"{name}_se_scale")([x, se]) | |
| x = tf.keras.layers.Conv2D( | |
| filters_out, | |
| 1, | |
| padding="same", | |
| use_bias=False, | |
| kernel_initializer="he_normal", | |
| name=f"{name}_project_conv", | |
| )(x) | |
| x = tf.keras.layers.BatchNormalization( | |
| momentum=0.99, epsilon=1e-3, name=f"{name}_project_bn" | |
| )(x) | |
| if stride == 1 and filters_in == filters_out: | |
| if drop_rate > 0: | |
| x = tf.keras.layers.Dropout( | |
| drop_rate, | |
| noise_shape=(None, 1, 1, 1), | |
| name=f"{name}_drop_path", | |
| )(x) | |
| x = tf.keras.layers.Add(name=f"{name}_add")([residual, x]) | |
| return x | |
| class SpatialTransformerBlock(tf.keras.layers.Layer): | |
| """Global spatial MHSA with depthwise positional mixing and an MLP.""" | |
| def __init__(self, channels: int, heads: int = 4, mlp_ratio: int = 4, | |
| drop_rate: float = 0.0, | |
| weight_fake_quant_bits: int | None = None, **kwargs): | |
| super().__init__(**kwargs) | |
| if channels % heads: | |
| raise ValueError(f"channels={channels} must be divisible by heads={heads}") | |
| if weight_fake_quant_bits not in (None, 4, 8): | |
| raise ValueError("weight_fake_quant_bits must be None, 4, or 8") | |
| self.channels = channels | |
| self.heads = heads | |
| self.mlp_ratio = mlp_ratio | |
| self.drop_rate = drop_rate | |
| self.weight_fake_quant_bits = weight_fake_quant_bits | |
| self.position_conv = tf.keras.layers.DepthwiseConv2D( | |
| 3, padding="same", name="position_conv" | |
| ) | |
| self.attention_norm = tf.keras.layers.LayerNormalization( | |
| epsilon=1e-6, name="attention_norm" | |
| ) | |
| self.attention = tf.keras.layers.MultiHeadAttention( | |
| num_heads=heads, | |
| key_dim=channels // heads, | |
| dropout=drop_rate, | |
| name="mhsa", | |
| ) | |
| self.attention_dropout = tf.keras.layers.Dropout(drop_rate) | |
| self.mlp_norm = tf.keras.layers.LayerNormalization(epsilon=1e-6, name="mlp_norm") | |
| self.mlp_expand = tf.keras.layers.Dense( | |
| channels * mlp_ratio, activation=tf.nn.relu6, name="mlp_expand" | |
| ) | |
| self.mlp_dropout = tf.keras.layers.Dropout(drop_rate) | |
| self.mlp_project = tf.keras.layers.Dense(channels, name="mlp_project") | |
| def call(self, inputs, training=None): | |
| bits = self.weight_fake_quant_bits | |
| position = _depthwise_conv_with_fake_quant(self.position_conv, inputs, bits) | |
| x = tf.math.add(inputs, position, name="position_residual_add") | |
| spatial_shape = tf.shape(x) | |
| tokens = tf.reshape(x, [spatial_shape[0], -1, self.channels]) | |
| attention_input = self.attention_norm(tokens) | |
| if bits is None: | |
| attended = self.attention( | |
| attention_input, attention_input, training=training | |
| ) | |
| else: | |
| if not self.attention.built: | |
| self.attention.build(attention_input.shape, attention_input.shape) | |
| query = _einsum_dense_with_fake_quant( | |
| self.attention._query_dense, | |
| attention_input, | |
| bits, | |
| flatten_output_dimensions=True, | |
| ) | |
| key = _einsum_dense_with_fake_quant( | |
| self.attention._key_dense, | |
| attention_input, | |
| bits, | |
| flatten_output_dimensions=True, | |
| ) | |
| value = _einsum_dense_with_fake_quant( | |
| self.attention._value_dense, | |
| attention_input, | |
| bits, | |
| flatten_output_dimensions=True, | |
| ) | |
| attended_heads, _ = self.attention._compute_attention( | |
| query, key, value, None, training, False | |
| ) | |
| attended = _einsum_dense_with_fake_quant( | |
| self.attention._output_dense, attended_heads, bits | |
| ) | |
| attended = self.attention_dropout(attended, training=training) | |
| tokens = tf.math.add(tokens, attended, name="attention_residual_add") | |
| mlp = self.mlp_norm(tokens) | |
| mlp = _dense_with_fake_quant(self.mlp_expand, mlp, bits) | |
| mlp = self.mlp_dropout(mlp, training=training) | |
| tokens = tf.math.add( | |
| tokens, _dense_with_fake_quant(self.mlp_project, mlp, bits), | |
| name="mlp_residual_add" | |
| ) | |
| output = tf.reshape(tokens, spatial_shape) | |
| return tf.ensure_shape(output, inputs.shape) | |
| def fake_quantized_kernel_parameters(self) -> int: | |
| if self.weight_fake_quant_bits is None: | |
| return 0 | |
| count = _kernel_parameter_count(self.position_conv) | |
| if self.attention.built: | |
| count += sum( | |
| _kernel_parameter_count(projection) | |
| for projection in ( | |
| self.attention._query_dense, | |
| self.attention._key_dense, | |
| self.attention._value_dense, | |
| self.attention._output_dense, | |
| ) | |
| ) | |
| count += _kernel_parameter_count(self.mlp_expand) | |
| count += _kernel_parameter_count(self.mlp_project) | |
| return count | |
| def get_config(self): | |
| config = super().get_config() | |
| config.update( | |
| { | |
| "channels": self.channels, | |
| "heads": self.heads, | |
| "mlp_ratio": self.mlp_ratio, | |
| "drop_rate": self.drop_rate, | |
| "weight_fake_quant_bits": self.weight_fake_quant_bits, | |
| } | |
| ) | |
| return config | |
| def _backbone( | |
| variant_name: str, | |
| hybrid: bool = False, | |
| dcnv4: bool = False, | |
| dcnv4_main_weight_bits: int | None = None, | |
| ): | |
| if variant_name not in VARIANTS: | |
| raise ValueError(f"Unknown EfficientNet scale {variant_name!r}; use {tuple(VARIANTS)}") | |
| variant = VARIANTS[variant_name] | |
| inputs = tf.keras.Input( | |
| shape=(variant.image_size, variant.image_size, 3), name="image" | |
| ) | |
| stem_filters = round_filters(32, variant.width) | |
| x = _conv_bn_relu6(inputs, stem_filters, 3, 2, "stem") | |
| repeats_by_stage = [round_repeats(spec[3], variant.depth) for spec in STAGE_SPECS] | |
| total_blocks = sum(repeats_by_stage) | |
| block_index = 0 | |
| feature_p3 = feature_p4 = feature_p5 = None | |
| filters_in = stem_filters | |
| for stage_index, (kernel, expansion, base_filters, _, first_stride) in enumerate( | |
| STAGE_SPECS | |
| ): | |
| filters_out = round_filters(base_filters, variant.width) | |
| repeats = repeats_by_stage[stage_index] | |
| if hybrid and stage_index == 5: | |
| x = _conv_bn_relu6( | |
| x, filters_out, 3, first_stride, "hybrid_stage6_patch" | |
| ) | |
| for block in range(repeats): | |
| drop_rate = 0.1 * block_index / max(total_blocks - 1, 1) | |
| x = SpatialTransformerBlock( | |
| filters_out, | |
| heads=4, | |
| mlp_ratio=4, | |
| drop_rate=drop_rate, | |
| name=f"hybrid_stage6_block{block + 1}", | |
| )(x) | |
| block_index += 1 | |
| elif hybrid and stage_index == 6: | |
| x = _conv_bn_relu6( | |
| x, filters_out, 1, first_stride, "hybrid_stage7_projection" | |
| ) | |
| for block in range(repeats): | |
| drop_rate = 0.1 * block_index / max(total_blocks - 1, 1) | |
| x = SpatialTransformerBlock( | |
| filters_out, | |
| heads=4, | |
| mlp_ratio=4, | |
| drop_rate=drop_rate, | |
| name=f"hybrid_stage7_block{block + 1}", | |
| )(x) | |
| block_index += 1 | |
| else: | |
| for block in range(repeats): | |
| stride = first_stride if block == 0 else 1 | |
| drop_rate = 0.1 * block_index / max(total_blocks - 1, 1) | |
| x = _mbconv( | |
| x, | |
| filters_in=filters_in, | |
| filters_out=filters_out, | |
| kernel=kernel, | |
| stride=stride, | |
| expansion=expansion, | |
| se_ratio=0.25, | |
| drop_rate=drop_rate, | |
| name=f"stage{stage_index + 1}_block{block + 1}", | |
| dcnv4_spatial=dcnv4 and stage_index == 6, | |
| dcnv4_main_weight_bits=dcnv4_main_weight_bits, | |
| ) | |
| filters_in = filters_out | |
| block_index += 1 | |
| if stage_index == 2: | |
| feature_p3 = x | |
| elif stage_index == 4: | |
| feature_p4 = x | |
| elif stage_index == 6: | |
| feature_p5 = x | |
| head_filters = round_filters(1280, variant.width) | |
| head = _conv_bn_relu6(x, head_filters, 1, 1, "head") | |
| return inputs, head, (feature_p3, feature_p4, feature_p5), variant | |
| def build_efficientnet(variant: str, num_classes: int = 1_000): | |
| inputs, features, _, scale = _backbone(variant) | |
| x = tf.keras.layers.GlobalAveragePooling2D(name="global_pool")(features) | |
| x = tf.keras.layers.Dropout(scale.dropout, name="classifier_dropout")(x) | |
| logits = tf.keras.layers.Dense(num_classes, dtype="float32", name="logits")(x) | |
| return tf.keras.Model(inputs, logits, name=f"efficientnet_{variant}_relu6") | |
| def build_efficientnet_dcnv4( | |
| variant: str, | |
| num_classes: int = 1_000, | |
| main_weight_bits: int | None = None, | |
| ): | |
| inputs, features, _, scale = _backbone( | |
| variant, dcnv4=True, dcnv4_main_weight_bits=main_weight_bits | |
| ) | |
| x = tf.keras.layers.GlobalAveragePooling2D(name="global_pool")(features) | |
| x = tf.keras.layers.Dropout(scale.dropout, name="classifier_dropout")(x) | |
| logits = tf.keras.layers.Dense(num_classes, dtype="float32", name="logits")(x) | |
| return tf.keras.Model( | |
| inputs, logits, name=f"efficientnet_{variant}_dcnv4_relu6" | |
| ) | |
| class FastNormalizedFusion(tf.keras.layers.Layer): | |
| """Learn nonnegative, normalized weights for a small set of feature maps.""" | |
| def __init__(self, inputs_count: int, epsilon: float = 1e-4, **kwargs): | |
| super().__init__(**kwargs) | |
| self.inputs_count = inputs_count | |
| self.epsilon = epsilon | |
| def build(self, input_shape): | |
| self.edge_weights = self.add_weight( | |
| name="edge_weights", | |
| shape=(self.inputs_count,), | |
| initializer="ones", | |
| trainable=True, | |
| ) | |
| super().build(input_shape) | |
| def call(self, inputs): | |
| weights = tf.nn.relu(self.edge_weights) | |
| weights = weights / (tf.reduce_sum(weights) + self.epsilon) | |
| return tf.add_n([weights[i] * value for i, value in enumerate(inputs)]) | |
| def get_config(self): | |
| config = super().get_config() | |
| config.update({"inputs_count": self.inputs_count, "epsilon": self.epsilon}) | |
| return config | |
| class BiFPNLayer(tf.keras.layers.Layer): | |
| def __init__( | |
| self, | |
| channels: int = 160, | |
| attention: bool = False, | |
| weight_fake_quant_bits: int | None = None, | |
| **kwargs, | |
| ): | |
| super().__init__(**kwargs) | |
| if weight_fake_quant_bits not in (None, 4, 8): | |
| raise ValueError("weight_fake_quant_bits must be None, 4, or 8") | |
| self.channels = channels | |
| self.attention_enabled = attention | |
| self.weight_fake_quant_bits = weight_fake_quant_bits | |
| self.projections = [ | |
| tf.keras.layers.Conv2D(channels, 1, padding="same", use_bias=False, name=f"input_proj_{i}") | |
| for i in range(3) | |
| ] | |
| self.projection_norms = [ | |
| tf.keras.layers.BatchNormalization(momentum=0.99, epsilon=1e-3, name=f"input_bn_{i}") | |
| for i in range(3) | |
| ] | |
| self.td4_fusion = FastNormalizedFusion(2, name="td4_fusion") | |
| self.td3_fusion = FastNormalizedFusion(2, name="td3_fusion") | |
| self.out4_fusion = FastNormalizedFusion(3, name="out4_fusion") | |
| self.out5_fusion = FastNormalizedFusion(2, name="out5_fusion") | |
| self.convs = [ | |
| tf.keras.layers.SeparableConv2D( | |
| channels, 3, padding="same", use_bias=False, name=f"node_conv_{i}" | |
| ) | |
| for i in range(4) | |
| ] | |
| self.norms = [ | |
| tf.keras.layers.BatchNormalization(momentum=0.99, epsilon=1e-3, name=f"node_bn_{i}") | |
| for i in range(4) | |
| ] | |
| self.down3 = tf.keras.layers.SeparableConv2D( | |
| channels, 3, strides=2, padding="same", use_bias=False, name="downsample_p3" | |
| ) | |
| self.down4 = tf.keras.layers.SeparableConv2D( | |
| channels, 3, strides=2, padding="same", use_bias=False, name="downsample_p4" | |
| ) | |
| self.p4_attention = ( | |
| SpatialTransformerBlock( | |
| channels, | |
| heads=4, | |
| mlp_ratio=2, | |
| weight_fake_quant_bits=weight_fake_quant_bits, | |
| name="p4_attention", | |
| ) | |
| if attention | |
| else None | |
| ) | |
| self.p5_attention = ( | |
| SpatialTransformerBlock( | |
| channels, | |
| heads=4, | |
| mlp_ratio=2, | |
| weight_fake_quant_bits=weight_fake_quant_bits, | |
| name="p5_attention", | |
| ) | |
| if attention | |
| else None | |
| ) | |
| def _node(self, value, index, training): | |
| value = _separable_conv_with_fake_quant( | |
| self.convs[index], value, self.weight_fake_quant_bits | |
| ) | |
| value = self.norms[index](value, training=training) | |
| return tf.nn.relu6(value) | |
| def call(self, inputs, training=None): | |
| p3, p4, p5 = inputs | |
| p3, p4, p5 = [ | |
| tf.nn.relu6( | |
| norm( | |
| _conv2d_with_fake_quant( | |
| projection, feature, self.weight_fake_quant_bits | |
| ), | |
| training=training, | |
| ) | |
| ) | |
| for projection, norm, feature in zip( | |
| self.projections, self.projection_norms, (p3, p4, p5) | |
| ) | |
| ] | |
| p5_up = tf.image.resize(p5, tf.shape(p4)[1:3], method="nearest") | |
| td4 = self._node(self.td4_fusion([p4, p5_up]), 0, training) | |
| td4_up = tf.image.resize(td4, tf.shape(p3)[1:3], method="nearest") | |
| td3 = self._node(self.td3_fusion([p3, td4_up]), 1, training) | |
| out4 = self._node( | |
| self.out4_fusion( | |
| [ | |
| p4, | |
| td4, | |
| _separable_conv_with_fake_quant( | |
| self.down3, td3, self.weight_fake_quant_bits | |
| ), | |
| ] | |
| ), | |
| 2, | |
| training, | |
| ) | |
| out5 = self._node( | |
| self.out5_fusion( | |
| [ | |
| p5, | |
| _separable_conv_with_fake_quant( | |
| self.down4, out4, self.weight_fake_quant_bits | |
| ), | |
| ] | |
| ), | |
| 3, | |
| training, | |
| ) | |
| if self.attention_enabled: | |
| out4 = self.p4_attention(out4, training=training) | |
| out5 = self.p5_attention(out5, training=training) | |
| return td3, out4, out5 | |
| def fake_quantized_kernel_parameters(self) -> int: | |
| if self.weight_fake_quant_bits is None: | |
| return 0 | |
| count = sum(_kernel_parameter_count(layer) for layer in self.projections) | |
| for layer in (*self.convs, self.down3, self.down4): | |
| count += _kernel_parameter_count(layer) | |
| if self.attention_enabled: | |
| count += self.p4_attention.fake_quantized_kernel_parameters() | |
| count += self.p5_attention.fake_quantized_kernel_parameters() | |
| return count | |
| def get_config(self): | |
| config = super().get_config() | |
| config.update( | |
| { | |
| "channels": self.channels, | |
| "attention": self.attention_enabled, | |
| "weight_fake_quant_bits": self.weight_fake_quant_bits, | |
| } | |
| ) | |
| return config | |
| def build_efficientnet_bifpn( | |
| variant: str, | |
| num_classes: int = 1_000, | |
| bifpn_repeats: int = 2, | |
| channels: int = 160, | |
| attention: bool = False, | |
| ): | |
| inputs, _, features, scale = _backbone(variant) | |
| for repeat in range(bifpn_repeats): | |
| features = BiFPNLayer( | |
| channels=channels, | |
| attention=attention, | |
| name=f"bifpn_{repeat + 1}", | |
| )(features) | |
| pooled = [ | |
| tf.keras.layers.GlobalAveragePooling2D(name=f"bifpn_pool_p{level}")(feature) | |
| for level, feature in zip((3, 4, 5), features) | |
| ] | |
| x = tf.keras.layers.Concatenate(name="bifpn_multiscale_pool")(pooled) | |
| x = tf.keras.layers.Dropout(scale.dropout, name="classifier_dropout")(x) | |
| logits = tf.keras.layers.Dense(num_classes, dtype="float32", name="logits")(x) | |
| neck_name = "bifpn_attention" if attention else "bifpn" | |
| model_name = f"efficientnet_{variant}_{neck_name}_relu6" | |
| return tf.keras.Model(inputs, logits, name=model_name) | |
| def build_hybrid(variant: str, num_classes: int = 1_000): | |
| inputs, features, _, scale = _backbone(variant, hybrid=True) | |
| x = tf.keras.layers.GlobalAveragePooling2D(name="global_pool")(features) | |
| x = tf.keras.layers.Dropout(scale.dropout, name="classifier_dropout")(x) | |
| logits = tf.keras.layers.Dense(num_classes, dtype="float32", name="logits")(x) | |
| return tf.keras.Model(inputs, logits, name=f"efficientnet_{variant}_mhsa_hybrid") | |
| def build_hybrid_bifpn( | |
| variant: str, num_classes: int = 1_000, bifpn_repeats: int = 2, channels: int = 160 | |
| ): | |
| inputs, _, features, scale = _backbone(variant, hybrid=True) | |
| for repeat in range(bifpn_repeats): | |
| features = BiFPNLayer( | |
| channels=channels, | |
| name=f"hybrid_bifpn_neck_{repeat + 1}", | |
| )(features) | |
| pooled = [ | |
| tf.keras.layers.GlobalAveragePooling2D(name=f"hybrid_bifpn_pool_p{level}")(feature) | |
| for level, feature in zip((3, 4, 5), features) | |
| ] | |
| x = tf.keras.layers.Concatenate(name="hybrid_bifpn_multiscale_pool")(pooled) | |
| x = tf.keras.layers.Dropout(scale.dropout, name="classifier_dropout")(x) | |
| logits = tf.keras.layers.Dense(num_classes, dtype="float32", name="logits")(x) | |
| return tf.keras.Model(inputs, logits, name=f"efficientnet_{variant}_mhsa_bifpn") | |
| def build_model(model_name: str, variant: str | None = None, num_classes: int = 1_000): | |
| """Build one registered model by name for training and quantization tools.""" | |
| if model_name.startswith("dcnv4_"): | |
| scale_name = variant or model_name.removeprefix("dcnv4_") | |
| return build_efficientnet_dcnv4(scale_name, num_classes) | |
| if model_name.startswith("hybrid_bifpn_"): | |
| scale_name = variant or model_name.removeprefix("hybrid_bifpn_") | |
| return build_hybrid_bifpn(scale_name, num_classes) | |
| if model_name.startswith("hybrid_"): | |
| scale_name = variant or model_name.removeprefix("hybrid_") | |
| return build_hybrid(scale_name, num_classes) | |
| if model_name.startswith("bifpn_"): | |
| attention = model_name.startswith("bifpn_attention_") | |
| prefix = "bifpn_attention_" if attention else "bifpn_" | |
| scale_name = variant or model_name.removeprefix(prefix) | |
| return build_efficientnet_bifpn(scale_name, num_classes, attention=attention) | |
| scale_name = variant or model_name | |
| return build_efficientnet(scale_name, num_classes) | |