"""TensorFlow/Keras EfficientNet, BiFPN, and CNN-attention classifiers. All convolutional blocks are implemented here rather than loaded from a pretrained application model. The default activation is ReLU6 for the planned integer-quantization comparisons. """ from __future__ import annotations from dataclasses import dataclass import math import tensorflow as tf @dataclass(frozen=True) class Variant: width: float depth: float image_size: int dropout: float VARIANTS = { "b0": Variant(width=1.0, depth=1.0, image_size=224, dropout=0.2), "b1": Variant(width=1.0, depth=1.1, image_size=240, dropout=0.2), "b2": Variant(width=1.1, depth=1.2, image_size=260, dropout=0.3), } STAGE_SPECS = ( # kernel, expansion, channels, repeats, first stride (3, 1, 16, 1, 1), (3, 6, 24, 2, 2), (5, 6, 40, 2, 2), (3, 6, 80, 3, 2), (5, 6, 112, 3, 1), (5, 6, 192, 4, 2), (3, 6, 320, 1, 1), ) def round_filters(filters: int, width: float, divisor: int = 8) -> int: scaled = filters * width rounded = max(divisor, int(scaled + divisor / 2) // divisor * divisor) if rounded < 0.9 * scaled: rounded += divisor return int(rounded) def round_repeats(repeats: int, depth: float) -> int: return int(math.ceil(repeats * depth)) def _conv_bn_relu6(x, filters, kernel, stride, name, activation=True): x = tf.keras.layers.Conv2D( filters, kernel, strides=stride, padding="same", use_bias=False, kernel_initializer="he_normal", name=f"{name}_conv", )(x) x = tf.keras.layers.BatchNormalization( momentum=0.99, epsilon=1e-3, name=f"{name}_bn" )(x) if activation: x = tf.keras.layers.ReLU(max_value=6.0, name=f"{name}_relu6")(x) return x def _fake_quantize_kernel_per_output_channel(kernel, num_bits: int): """Apply symmetric, narrow-range fake quantization to a weight tensor.""" kernel32 = tf.cast(kernel, tf.float32) reduction_axes = tuple(range(kernel32.shape.rank - 1)) bound = tf.reduce_max(tf.abs(kernel32), axis=reduction_axes) bound = tf.stop_gradient(tf.maximum(bound, 1e-8)) return tf.quantization.fake_quant_with_min_max_vars_per_channel( kernel32, min=-bound, max=bound, num_bits=num_bits, narrow_range=True, ) def _conv2d_with_fake_quant(layer, inputs, num_bits: int | None): compute_dtype = tf.as_dtype(layer.compute_dtype) inputs = tf.cast(inputs, compute_dtype) if num_bits is None: return layer(inputs) if not layer.built: layer.build(inputs.shape) kernel = _fake_quantize_kernel_per_output_channel(layer.kernel, num_bits) kernel = tf.cast(kernel, compute_dtype) output = tf.nn.conv2d( inputs, kernel, strides=[1, *layer.strides, 1], padding=layer.padding.upper(), data_format="NHWC", dilations=[1, *layer.dilation_rate, 1], ) if layer.use_bias: output = tf.nn.bias_add( output, tf.cast(layer.bias, compute_dtype), data_format="NHWC" ) if layer.activation is not None: output = layer.activation(output) return output def _depthwise_conv_with_fake_quant(layer, inputs, num_bits: int | None): if num_bits is None: return layer(inputs) if not layer.built: layer.build(inputs.shape) kernel32 = tf.cast(layer.kernel, tf.float32) input_channels = int(kernel32.shape[-2]) output_channels = input_channels * layer.depth_multiplier flattened = tf.reshape( kernel32, [layer.kernel_size[0], layer.kernel_size[1], output_channels] ) quantized = _fake_quantize_kernel_per_output_channel(flattened, num_bits) kernel = tf.cast(tf.reshape(quantized, tf.shape(kernel32)), layer.compute_dtype) inputs = tf.cast(inputs, layer.compute_dtype) output = tf.keras.ops.depthwise_conv( inputs, kernel, strides=layer.strides, padding=layer.padding, dilation_rate=layer.dilation_rate, data_format=layer.data_format, ) if layer.use_bias: output = tf.nn.bias_add( output, tf.cast(layer.bias, layer.compute_dtype), data_format="NHWC" ) if layer.activation is not None: output = layer.activation(output) return output def _separable_conv_with_fake_quant(layer, inputs, num_bits: int | None): if num_bits is None: return layer(inputs) if not layer.built: layer.build(inputs.shape) depthwise32 = tf.cast(layer.depthwise_kernel, tf.float32) output_channels = int(depthwise32.shape[-2]) * layer.depth_multiplier flattened = tf.reshape( depthwise32, [layer.kernel_size[0], layer.kernel_size[1], output_channels], ) depthwise = _fake_quantize_kernel_per_output_channel(flattened, num_bits) depthwise = tf.cast(tf.reshape(depthwise, tf.shape(depthwise32)), layer.compute_dtype) pointwise = _fake_quantize_kernel_per_output_channel( layer.pointwise_kernel, num_bits ) pointwise = tf.cast(pointwise, layer.compute_dtype) inputs = tf.cast(inputs, layer.compute_dtype) output = tf.keras.ops.separable_conv( inputs, depthwise, pointwise, strides=layer.strides, padding=layer.padding, data_format=layer.data_format, dilation_rate=layer.dilation_rate, ) if layer.use_bias: output = tf.nn.bias_add( output, tf.cast(layer.bias, layer.compute_dtype), data_format="NHWC" ) if layer.activation is not None: output = layer.activation(output) return output def _dense_with_fake_quant(layer, inputs, num_bits: int | None): if num_bits is None: return layer(inputs) if not layer.built: layer.build(inputs.shape) kernel = _fake_quantize_kernel_per_output_channel(layer.kernel, num_bits) output = tf.keras.ops.matmul( tf.cast(inputs, layer.compute_dtype), tf.cast(kernel, layer.compute_dtype) ) if layer.bias is not None: output = tf.keras.ops.add(output, tf.cast(layer.bias, layer.compute_dtype)) if layer.activation is not None: output = layer.activation(output) return output def _einsum_dense_with_fake_quant( layer, inputs, num_bits: int, *, flatten_output_dimensions: bool = False ): if not layer.built: layer.build(inputs.shape) kernel32 = tf.cast(layer.kernel, tf.float32) if flatten_output_dimensions: original_shape = tf.shape(kernel32) flat_shape = tf.stack([original_shape[0], -1]) flattened = tf.reshape(kernel32, flat_shape) quantized = _fake_quantize_kernel_per_output_channel(flattened, num_bits) kernel = tf.reshape(quantized, original_shape) else: kernel = _fake_quantize_kernel_per_output_channel(kernel32, num_bits) output = tf.keras.ops.einsum( layer.equation, tf.cast(inputs, layer.compute_dtype), tf.cast(kernel, layer.compute_dtype), ) if layer.bias is not None: output = tf.keras.ops.add(output, tf.cast(layer.bias, layer.compute_dtype)) if layer.activation is not None: output = layer.activation(output) return output def _kernel_parameter_count(layer) -> int: count = 0 for name in ("kernel", "depthwise_kernel", "pointwise_kernel"): kernel = getattr(layer, name, None) if kernel is not None: count += int(tf.TensorShape(kernel.shape).num_elements() or 0) return count @tf.keras.utils.register_keras_serializable(package="imagenet_research") class DCNv4Reference(tf.keras.layers.Layer): """Differentiable TensorFlow reference for grouped DCNv4 aggregation. This implements the published operator's value projection, learned offsets and unnormalized aggregation weights, bilinear sampling, and output projection. It is a portable reference implementation; it does not include OpenGVLab's fused CUDA kernel or its memory-access optimizations. """ def __init__( self, channels: int, groups: int = 8, kernel_size: int = 3, offset_scale: float = 1.0, main_weight_bits: int | None = None, **kwargs, ): super().__init__(**kwargs) if channels % groups: raise ValueError(f"channels={channels} must be divisible by groups={groups}") if kernel_size < 1 or kernel_size % 2 == 0: raise ValueError("kernel_size must be a positive odd number") if main_weight_bits not in (None, 4, 8): raise ValueError("main_weight_bits must be None, 4, or 8") self.channels = channels self.groups = groups self.kernel_size = kernel_size self.offset_scale = float(offset_scale) self.main_weight_bits = main_weight_bits self.points = kernel_size * kernel_size self.value_proj = tf.keras.layers.Conv2D( channels, 1, use_bias=True, kernel_initializer="glorot_uniform", name="value_proj" ) self.offset_mask_projection = tf.keras.layers.Conv2D( groups * self.points * 3, 1, use_bias=True, kernel_initializer="zeros", bias_initializer="zeros", dtype="float32", name="offset_mask_projection", ) self.output_proj = tf.keras.layers.Conv2D( channels, 1, use_bias=True, kernel_initializer="glorot_uniform", name="output_proj" ) def build(self, input_shape): if input_shape[-1] is not None and int(input_shape[-1]) != self.channels: raise ValueError( f"DCNv4 channels={self.channels} does not match input channels={input_shape[-1]}" ) self.value_proj.build(input_shape) self.offset_mask_projection.build(input_shape) self.output_proj.build(input_shape) super().build(input_shape) def _aggregate(self, values, offsets, aggregation_weights): shape = tf.shape(values) batch, height, width = shape[0], shape[1], shape[2] channels_per_group = self.channels // self.groups padding = self.kernel_size // 2 # OpenGVLab enumerates sampling points with width outermost; keep this # order aligned with the predictor's per-point offset and weight slots. kernel_offsets = tf.constant( [ (row - padding, column - padding) for column in range(self.kernel_size) for row in range(self.kernel_size) ], dtype=tf.float32, ) grid_y = tf.cast(tf.range(height), tf.float32)[None, :, None, None, None] grid_x = tf.cast(tf.range(width), tf.float32)[None, None, :, None, None] point_y = kernel_offsets[:, 0][None, None, None, None, :] point_x = kernel_offsets[:, 1][None, None, None, None, :] # The official DCNv4 predictor stores each displacement as (x, y), # then scales both the regular kernel lattice and learned displacement. sample_y = ( grid_y + point_y * self.offset_scale + offsets[..., 1] * self.offset_scale ) sample_x = ( grid_x + point_x * self.offset_scale + offsets[..., 0] * self.offset_scale ) y0_float = tf.floor(sample_y) x0_float = tf.floor(sample_x) y0 = tf.cast(y0_float, tf.int32) x0 = tf.cast(x0_float, tf.int32) y1 = y0 + 1 x1 = x0 + 1 fraction_y = sample_y - y0_float fraction_x = sample_x - x0_float grouped_values = tf.reshape( values, [batch, height, width, self.groups, channels_per_group], ) grouped_values = tf.transpose(grouped_values, [0, 3, 1, 2, 4]) grouped_values = tf.reshape( grouped_values, [batch * self.groups, height * width, channels_per_group], ) def gather_neighbour(y_index, x_index): valid = ( (y_index >= 0) & (y_index < height) & (x_index >= 0) & (x_index < width) ) flat_index = tf.clip_by_value(y_index, 0, height - 1) * width flat_index += tf.clip_by_value(x_index, 0, width - 1) flat_index = tf.transpose(flat_index, [0, 3, 1, 2, 4]) valid = tf.transpose(valid, [0, 3, 1, 2, 4]) flat_index = tf.reshape(flat_index, [batch * self.groups, height * width * self.points]) valid = tf.reshape(valid, [batch * self.groups, height * width * self.points]) samples = tf.gather( grouped_values, flat_index, axis=1, batch_dims=1 ) samples = tf.reshape( samples, [batch, self.groups, height * width, self.points, channels_per_group], ) valid = tf.reshape( valid, [batch, self.groups, height * width, self.points, 1] ) return samples * tf.cast(valid, samples.dtype) fraction_y = tf.transpose(fraction_y, [0, 3, 1, 2, 4]) fraction_x = tf.transpose(fraction_x, [0, 3, 1, 2, 4]) fraction_y = tf.reshape(fraction_y, [batch, self.groups, height * width, self.points]) fraction_x = tf.reshape(fraction_x, [batch, self.groups, height * width, self.points]) weights = tf.transpose(aggregation_weights, [0, 3, 1, 2, 4]) weights = tf.reshape(weights, [batch, self.groups, height * width, self.points]) top_left = gather_neighbour(y0, x0) top_right = gather_neighbour(y0, x1) bottom_left = gather_neighbour(y1, x0) bottom_right = gather_neighbour(y1, x1) sampled = ( top_left * ((1.0 - fraction_y) * (1.0 - fraction_x))[..., None] + top_right * ((1.0 - fraction_y) * fraction_x)[..., None] + bottom_left * (fraction_y * (1.0 - fraction_x))[..., None] + bottom_right * (fraction_y * fraction_x)[..., None] ) aggregated = tf.reduce_sum(sampled * weights[..., None], axis=3) aggregated = tf.transpose(aggregated, [0, 2, 1, 3]) return tf.reshape(aggregated, [batch, height, width, self.channels]) def call(self, inputs): original_dtype = inputs.dtype inputs32 = tf.cast(inputs, tf.float32) values = _conv2d_with_fake_quant( self.value_proj, inputs32, self.main_weight_bits ) offset_mask = self.offset_mask_projection(inputs32) shape = tf.shape(offset_mask) offset_mask = tf.reshape( offset_mask, [shape[0], shape[1], shape[2], self.groups, self.points * 3], ) # DCNv4 packs each group's 2*K x/y offsets first, followed by K # aggregation weights. Keep this ordering aligned with the official # kernel's group stride and its p_offset_ptr + 2*K weight offset. offsets = tf.reshape( offset_mask[..., : self.points * 2], [shape[0], shape[1], shape[2], self.groups, self.points, 2], ) aggregation_weights = offset_mask[..., self.points * 2 :] output = self._aggregate( tf.cast(values, tf.float32), offsets, aggregation_weights ) output = _conv2d_with_fake_quant( self.output_proj, output, self.main_weight_bits ) return tf.cast(output, original_dtype) def compute_output_shape(self, input_shape): return input_shape def get_config(self): config = super().get_config() config.update( { "channels": self.channels, "groups": self.groups, "kernel_size": self.kernel_size, "offset_scale": self.offset_scale, "main_weight_bits": self.main_weight_bits, } ) return config def _mbconv( x, filters_in, filters_out, kernel, stride, expansion, se_ratio, drop_rate, name, dcnv4_spatial: bool = False, dcnv4_main_weight_bits: int | None = None, ): residual = x expanded_filters = filters_in * expansion if expansion != 1: x = _conv_bn_relu6( x, expanded_filters, 1, 1, f"{name}_expand", activation=True ) if dcnv4_spatial: if stride != 1 or kernel != 3: raise ValueError("The DCNv4 reference path currently supports stride-1 3x3 blocks") x = DCNv4Reference( channels=expanded_filters, groups=8, kernel_size=3, main_weight_bits=dcnv4_main_weight_bits, name=f"{name}_dcnv4", )(x) else: x = tf.keras.layers.DepthwiseConv2D( kernel, strides=stride, padding="same", use_bias=False, depthwise_initializer="he_normal", name=f"{name}_depthwise", )(x) x = tf.keras.layers.BatchNormalization( momentum=0.99, epsilon=1e-3, name=f"{name}_depthwise_bn" )(x) x = tf.keras.layers.ReLU(max_value=6.0, name=f"{name}_depthwise_relu6")(x) # The SE gate acts on the expanded tensor, but its squeeze width is based # on the MBConv input channels. squeeze_filters = max(1, int(filters_in * se_ratio)) se = tf.keras.layers.GlobalAveragePooling2D(keepdims=True, name=f"{name}_se_pool")(x) se = tf.keras.layers.Conv2D( squeeze_filters, 1, activation="relu", kernel_initializer="he_normal", name=f"{name}_se_reduce", )(se) se = tf.keras.layers.Conv2D( expanded_filters, 1, activation="sigmoid", kernel_initializer="glorot_uniform", name=f"{name}_se_expand", )(se) x = tf.keras.layers.Multiply(name=f"{name}_se_scale")([x, se]) x = tf.keras.layers.Conv2D( filters_out, 1, padding="same", use_bias=False, kernel_initializer="he_normal", name=f"{name}_project_conv", )(x) x = tf.keras.layers.BatchNormalization( momentum=0.99, epsilon=1e-3, name=f"{name}_project_bn" )(x) if stride == 1 and filters_in == filters_out: if drop_rate > 0: x = tf.keras.layers.Dropout( drop_rate, noise_shape=(None, 1, 1, 1), name=f"{name}_drop_path", )(x) x = tf.keras.layers.Add(name=f"{name}_add")([residual, x]) return x @tf.keras.utils.register_keras_serializable(package="imagenet_research") class SpatialTransformerBlock(tf.keras.layers.Layer): """Global spatial MHSA with depthwise positional mixing and an MLP.""" def __init__(self, channels: int, heads: int = 4, mlp_ratio: int = 4, drop_rate: float = 0.0, weight_fake_quant_bits: int | None = None, **kwargs): super().__init__(**kwargs) if channels % heads: raise ValueError(f"channels={channels} must be divisible by heads={heads}") if weight_fake_quant_bits not in (None, 4, 8): raise ValueError("weight_fake_quant_bits must be None, 4, or 8") self.channels = channels self.heads = heads self.mlp_ratio = mlp_ratio self.drop_rate = drop_rate self.weight_fake_quant_bits = weight_fake_quant_bits self.position_conv = tf.keras.layers.DepthwiseConv2D( 3, padding="same", name="position_conv" ) self.attention_norm = tf.keras.layers.LayerNormalization( epsilon=1e-6, name="attention_norm" ) self.attention = tf.keras.layers.MultiHeadAttention( num_heads=heads, key_dim=channels // heads, dropout=drop_rate, name="mhsa", ) self.attention_dropout = tf.keras.layers.Dropout(drop_rate) self.mlp_norm = tf.keras.layers.LayerNormalization(epsilon=1e-6, name="mlp_norm") self.mlp_expand = tf.keras.layers.Dense( channels * mlp_ratio, activation=tf.nn.relu6, name="mlp_expand" ) self.mlp_dropout = tf.keras.layers.Dropout(drop_rate) self.mlp_project = tf.keras.layers.Dense(channels, name="mlp_project") def call(self, inputs, training=None): bits = self.weight_fake_quant_bits position = _depthwise_conv_with_fake_quant(self.position_conv, inputs, bits) x = tf.math.add(inputs, position, name="position_residual_add") spatial_shape = tf.shape(x) tokens = tf.reshape(x, [spatial_shape[0], -1, self.channels]) attention_input = self.attention_norm(tokens) if bits is None: attended = self.attention( attention_input, attention_input, training=training ) else: if not self.attention.built: self.attention.build(attention_input.shape, attention_input.shape) query = _einsum_dense_with_fake_quant( self.attention._query_dense, attention_input, bits, flatten_output_dimensions=True, ) key = _einsum_dense_with_fake_quant( self.attention._key_dense, attention_input, bits, flatten_output_dimensions=True, ) value = _einsum_dense_with_fake_quant( self.attention._value_dense, attention_input, bits, flatten_output_dimensions=True, ) attended_heads, _ = self.attention._compute_attention( query, key, value, None, training, False ) attended = _einsum_dense_with_fake_quant( self.attention._output_dense, attended_heads, bits ) attended = self.attention_dropout(attended, training=training) tokens = tf.math.add(tokens, attended, name="attention_residual_add") mlp = self.mlp_norm(tokens) mlp = _dense_with_fake_quant(self.mlp_expand, mlp, bits) mlp = self.mlp_dropout(mlp, training=training) tokens = tf.math.add( tokens, _dense_with_fake_quant(self.mlp_project, mlp, bits), name="mlp_residual_add" ) output = tf.reshape(tokens, spatial_shape) return tf.ensure_shape(output, inputs.shape) def fake_quantized_kernel_parameters(self) -> int: if self.weight_fake_quant_bits is None: return 0 count = _kernel_parameter_count(self.position_conv) if self.attention.built: count += sum( _kernel_parameter_count(projection) for projection in ( self.attention._query_dense, self.attention._key_dense, self.attention._value_dense, self.attention._output_dense, ) ) count += _kernel_parameter_count(self.mlp_expand) count += _kernel_parameter_count(self.mlp_project) return count def get_config(self): config = super().get_config() config.update( { "channels": self.channels, "heads": self.heads, "mlp_ratio": self.mlp_ratio, "drop_rate": self.drop_rate, "weight_fake_quant_bits": self.weight_fake_quant_bits, } ) return config def _backbone( variant_name: str, hybrid: bool = False, dcnv4: bool = False, dcnv4_main_weight_bits: int | None = None, ): if variant_name not in VARIANTS: raise ValueError(f"Unknown EfficientNet scale {variant_name!r}; use {tuple(VARIANTS)}") variant = VARIANTS[variant_name] inputs = tf.keras.Input( shape=(variant.image_size, variant.image_size, 3), name="image" ) stem_filters = round_filters(32, variant.width) x = _conv_bn_relu6(inputs, stem_filters, 3, 2, "stem") repeats_by_stage = [round_repeats(spec[3], variant.depth) for spec in STAGE_SPECS] total_blocks = sum(repeats_by_stage) block_index = 0 feature_p3 = feature_p4 = feature_p5 = None filters_in = stem_filters for stage_index, (kernel, expansion, base_filters, _, first_stride) in enumerate( STAGE_SPECS ): filters_out = round_filters(base_filters, variant.width) repeats = repeats_by_stage[stage_index] if hybrid and stage_index == 5: x = _conv_bn_relu6( x, filters_out, 3, first_stride, "hybrid_stage6_patch" ) for block in range(repeats): drop_rate = 0.1 * block_index / max(total_blocks - 1, 1) x = SpatialTransformerBlock( filters_out, heads=4, mlp_ratio=4, drop_rate=drop_rate, name=f"hybrid_stage6_block{block + 1}", )(x) block_index += 1 elif hybrid and stage_index == 6: x = _conv_bn_relu6( x, filters_out, 1, first_stride, "hybrid_stage7_projection" ) for block in range(repeats): drop_rate = 0.1 * block_index / max(total_blocks - 1, 1) x = SpatialTransformerBlock( filters_out, heads=4, mlp_ratio=4, drop_rate=drop_rate, name=f"hybrid_stage7_block{block + 1}", )(x) block_index += 1 else: for block in range(repeats): stride = first_stride if block == 0 else 1 drop_rate = 0.1 * block_index / max(total_blocks - 1, 1) x = _mbconv( x, filters_in=filters_in, filters_out=filters_out, kernel=kernel, stride=stride, expansion=expansion, se_ratio=0.25, drop_rate=drop_rate, name=f"stage{stage_index + 1}_block{block + 1}", dcnv4_spatial=dcnv4 and stage_index == 6, dcnv4_main_weight_bits=dcnv4_main_weight_bits, ) filters_in = filters_out block_index += 1 if stage_index == 2: feature_p3 = x elif stage_index == 4: feature_p4 = x elif stage_index == 6: feature_p5 = x head_filters = round_filters(1280, variant.width) head = _conv_bn_relu6(x, head_filters, 1, 1, "head") return inputs, head, (feature_p3, feature_p4, feature_p5), variant def build_efficientnet(variant: str, num_classes: int = 1_000): inputs, features, _, scale = _backbone(variant) x = tf.keras.layers.GlobalAveragePooling2D(name="global_pool")(features) x = tf.keras.layers.Dropout(scale.dropout, name="classifier_dropout")(x) logits = tf.keras.layers.Dense(num_classes, dtype="float32", name="logits")(x) return tf.keras.Model(inputs, logits, name=f"efficientnet_{variant}_relu6") def build_efficientnet_dcnv4( variant: str, num_classes: int = 1_000, main_weight_bits: int | None = None, ): inputs, features, _, scale = _backbone( variant, dcnv4=True, dcnv4_main_weight_bits=main_weight_bits ) x = tf.keras.layers.GlobalAveragePooling2D(name="global_pool")(features) x = tf.keras.layers.Dropout(scale.dropout, name="classifier_dropout")(x) logits = tf.keras.layers.Dense(num_classes, dtype="float32", name="logits")(x) return tf.keras.Model( inputs, logits, name=f"efficientnet_{variant}_dcnv4_relu6" ) @tf.keras.utils.register_keras_serializable(package="imagenet_research") class FastNormalizedFusion(tf.keras.layers.Layer): """Learn nonnegative, normalized weights for a small set of feature maps.""" def __init__(self, inputs_count: int, epsilon: float = 1e-4, **kwargs): super().__init__(**kwargs) self.inputs_count = inputs_count self.epsilon = epsilon def build(self, input_shape): self.edge_weights = self.add_weight( name="edge_weights", shape=(self.inputs_count,), initializer="ones", trainable=True, ) super().build(input_shape) def call(self, inputs): weights = tf.nn.relu(self.edge_weights) weights = weights / (tf.reduce_sum(weights) + self.epsilon) return tf.add_n([weights[i] * value for i, value in enumerate(inputs)]) def get_config(self): config = super().get_config() config.update({"inputs_count": self.inputs_count, "epsilon": self.epsilon}) return config @tf.keras.utils.register_keras_serializable(package="imagenet_research") class BiFPNLayer(tf.keras.layers.Layer): def __init__( self, channels: int = 160, attention: bool = False, weight_fake_quant_bits: int | None = None, **kwargs, ): super().__init__(**kwargs) if weight_fake_quant_bits not in (None, 4, 8): raise ValueError("weight_fake_quant_bits must be None, 4, or 8") self.channels = channels self.attention_enabled = attention self.weight_fake_quant_bits = weight_fake_quant_bits self.projections = [ tf.keras.layers.Conv2D(channels, 1, padding="same", use_bias=False, name=f"input_proj_{i}") for i in range(3) ] self.projection_norms = [ tf.keras.layers.BatchNormalization(momentum=0.99, epsilon=1e-3, name=f"input_bn_{i}") for i in range(3) ] self.td4_fusion = FastNormalizedFusion(2, name="td4_fusion") self.td3_fusion = FastNormalizedFusion(2, name="td3_fusion") self.out4_fusion = FastNormalizedFusion(3, name="out4_fusion") self.out5_fusion = FastNormalizedFusion(2, name="out5_fusion") self.convs = [ tf.keras.layers.SeparableConv2D( channels, 3, padding="same", use_bias=False, name=f"node_conv_{i}" ) for i in range(4) ] self.norms = [ tf.keras.layers.BatchNormalization(momentum=0.99, epsilon=1e-3, name=f"node_bn_{i}") for i in range(4) ] self.down3 = tf.keras.layers.SeparableConv2D( channels, 3, strides=2, padding="same", use_bias=False, name="downsample_p3" ) self.down4 = tf.keras.layers.SeparableConv2D( channels, 3, strides=2, padding="same", use_bias=False, name="downsample_p4" ) self.p4_attention = ( SpatialTransformerBlock( channels, heads=4, mlp_ratio=2, weight_fake_quant_bits=weight_fake_quant_bits, name="p4_attention", ) if attention else None ) self.p5_attention = ( SpatialTransformerBlock( channels, heads=4, mlp_ratio=2, weight_fake_quant_bits=weight_fake_quant_bits, name="p5_attention", ) if attention else None ) def _node(self, value, index, training): value = _separable_conv_with_fake_quant( self.convs[index], value, self.weight_fake_quant_bits ) value = self.norms[index](value, training=training) return tf.nn.relu6(value) def call(self, inputs, training=None): p3, p4, p5 = inputs p3, p4, p5 = [ tf.nn.relu6( norm( _conv2d_with_fake_quant( projection, feature, self.weight_fake_quant_bits ), training=training, ) ) for projection, norm, feature in zip( self.projections, self.projection_norms, (p3, p4, p5) ) ] p5_up = tf.image.resize(p5, tf.shape(p4)[1:3], method="nearest") td4 = self._node(self.td4_fusion([p4, p5_up]), 0, training) td4_up = tf.image.resize(td4, tf.shape(p3)[1:3], method="nearest") td3 = self._node(self.td3_fusion([p3, td4_up]), 1, training) out4 = self._node( self.out4_fusion( [ p4, td4, _separable_conv_with_fake_quant( self.down3, td3, self.weight_fake_quant_bits ), ] ), 2, training, ) out5 = self._node( self.out5_fusion( [ p5, _separable_conv_with_fake_quant( self.down4, out4, self.weight_fake_quant_bits ), ] ), 3, training, ) if self.attention_enabled: out4 = self.p4_attention(out4, training=training) out5 = self.p5_attention(out5, training=training) return td3, out4, out5 def fake_quantized_kernel_parameters(self) -> int: if self.weight_fake_quant_bits is None: return 0 count = sum(_kernel_parameter_count(layer) for layer in self.projections) for layer in (*self.convs, self.down3, self.down4): count += _kernel_parameter_count(layer) if self.attention_enabled: count += self.p4_attention.fake_quantized_kernel_parameters() count += self.p5_attention.fake_quantized_kernel_parameters() return count def get_config(self): config = super().get_config() config.update( { "channels": self.channels, "attention": self.attention_enabled, "weight_fake_quant_bits": self.weight_fake_quant_bits, } ) return config def build_efficientnet_bifpn( variant: str, num_classes: int = 1_000, bifpn_repeats: int = 2, channels: int = 160, attention: bool = False, ): inputs, _, features, scale = _backbone(variant) for repeat in range(bifpn_repeats): features = BiFPNLayer( channels=channels, attention=attention, name=f"bifpn_{repeat + 1}", )(features) pooled = [ tf.keras.layers.GlobalAveragePooling2D(name=f"bifpn_pool_p{level}")(feature) for level, feature in zip((3, 4, 5), features) ] x = tf.keras.layers.Concatenate(name="bifpn_multiscale_pool")(pooled) x = tf.keras.layers.Dropout(scale.dropout, name="classifier_dropout")(x) logits = tf.keras.layers.Dense(num_classes, dtype="float32", name="logits")(x) neck_name = "bifpn_attention" if attention else "bifpn" model_name = f"efficientnet_{variant}_{neck_name}_relu6" return tf.keras.Model(inputs, logits, name=model_name) def build_hybrid(variant: str, num_classes: int = 1_000): inputs, features, _, scale = _backbone(variant, hybrid=True) x = tf.keras.layers.GlobalAveragePooling2D(name="global_pool")(features) x = tf.keras.layers.Dropout(scale.dropout, name="classifier_dropout")(x) logits = tf.keras.layers.Dense(num_classes, dtype="float32", name="logits")(x) return tf.keras.Model(inputs, logits, name=f"efficientnet_{variant}_mhsa_hybrid") def build_hybrid_bifpn( variant: str, num_classes: int = 1_000, bifpn_repeats: int = 2, channels: int = 160 ): inputs, _, features, scale = _backbone(variant, hybrid=True) for repeat in range(bifpn_repeats): features = BiFPNLayer( channels=channels, name=f"hybrid_bifpn_neck_{repeat + 1}", )(features) pooled = [ tf.keras.layers.GlobalAveragePooling2D(name=f"hybrid_bifpn_pool_p{level}")(feature) for level, feature in zip((3, 4, 5), features) ] x = tf.keras.layers.Concatenate(name="hybrid_bifpn_multiscale_pool")(pooled) x = tf.keras.layers.Dropout(scale.dropout, name="classifier_dropout")(x) logits = tf.keras.layers.Dense(num_classes, dtype="float32", name="logits")(x) return tf.keras.Model(inputs, logits, name=f"efficientnet_{variant}_mhsa_bifpn") def build_model(model_name: str, variant: str | None = None, num_classes: int = 1_000): """Build one registered model by name for training and quantization tools.""" if model_name.startswith("dcnv4_"): scale_name = variant or model_name.removeprefix("dcnv4_") return build_efficientnet_dcnv4(scale_name, num_classes) if model_name.startswith("hybrid_bifpn_"): scale_name = variant or model_name.removeprefix("hybrid_bifpn_") return build_hybrid_bifpn(scale_name, num_classes) if model_name.startswith("hybrid_"): scale_name = variant or model_name.removeprefix("hybrid_") return build_hybrid(scale_name, num_classes) if model_name.startswith("bifpn_"): attention = model_name.startswith("bifpn_attention_") prefix = "bifpn_attention_" if attention else "bifpn_" scale_name = variant or model_name.removeprefix(prefix) return build_efficientnet_bifpn(scale_name, num_classes, attention=attention) scale_name = variant or model_name return build_efficientnet(scale_name, num_classes)