meatloafer's picture
Upload verified model weights, LiteRT FlatBuffers, DP profiles, code, and model card
2abfed0 verified
Raw History Blame Contribute Delete
37.5 kB
"""TensorFlow/Keras EfficientNet, BiFPN, and CNN-attention classifiers.
All convolutional blocks are implemented here rather than loaded from a
pretrained application model. The default activation is ReLU6 for the planned
integer-quantization comparisons.
"""
from __future__ import annotations
from dataclasses import dataclass
import math
import tensorflow as tf
@dataclass(frozen=True)
class Variant:
width: float
depth: float
image_size: int
dropout: float
VARIANTS = {
"b0": Variant(width=1.0, depth=1.0, image_size=224, dropout=0.2),
"b1": Variant(width=1.0, depth=1.1, image_size=240, dropout=0.2),
"b2": Variant(width=1.1, depth=1.2, image_size=260, dropout=0.3),
}
STAGE_SPECS = (
# kernel, expansion, channels, repeats, first stride
(3, 1, 16, 1, 1),
(3, 6, 24, 2, 2),
(5, 6, 40, 2, 2),
(3, 6, 80, 3, 2),
(5, 6, 112, 3, 1),
(5, 6, 192, 4, 2),
(3, 6, 320, 1, 1),
)
def round_filters(filters: int, width: float, divisor: int = 8) -> int:
scaled = filters * width
rounded = max(divisor, int(scaled + divisor / 2) // divisor * divisor)
if rounded < 0.9 * scaled:
rounded += divisor
return int(rounded)
def round_repeats(repeats: int, depth: float) -> int:
return int(math.ceil(repeats * depth))
def _conv_bn_relu6(x, filters, kernel, stride, name, activation=True):
x = tf.keras.layers.Conv2D(
filters,
kernel,
strides=stride,
padding="same",
use_bias=False,
kernel_initializer="he_normal",
name=f"{name}_conv",
)(x)
x = tf.keras.layers.BatchNormalization(
momentum=0.99, epsilon=1e-3, name=f"{name}_bn"
)(x)
if activation:
x = tf.keras.layers.ReLU(max_value=6.0, name=f"{name}_relu6")(x)
return x
def _fake_quantize_kernel_per_output_channel(kernel, num_bits: int):
"""Apply symmetric, narrow-range fake quantization to a weight tensor."""
kernel32 = tf.cast(kernel, tf.float32)
reduction_axes = tuple(range(kernel32.shape.rank - 1))
bound = tf.reduce_max(tf.abs(kernel32), axis=reduction_axes)
bound = tf.stop_gradient(tf.maximum(bound, 1e-8))
return tf.quantization.fake_quant_with_min_max_vars_per_channel(
kernel32,
min=-bound,
max=bound,
num_bits=num_bits,
narrow_range=True,
)
def _conv2d_with_fake_quant(layer, inputs, num_bits: int | None):
compute_dtype = tf.as_dtype(layer.compute_dtype)
inputs = tf.cast(inputs, compute_dtype)
if num_bits is None:
return layer(inputs)
if not layer.built:
layer.build(inputs.shape)
kernel = _fake_quantize_kernel_per_output_channel(layer.kernel, num_bits)
kernel = tf.cast(kernel, compute_dtype)
output = tf.nn.conv2d(
inputs,
kernel,
strides=[1, *layer.strides, 1],
padding=layer.padding.upper(),
data_format="NHWC",
dilations=[1, *layer.dilation_rate, 1],
)
if layer.use_bias:
output = tf.nn.bias_add(
output, tf.cast(layer.bias, compute_dtype), data_format="NHWC"
)
if layer.activation is not None:
output = layer.activation(output)
return output
def _depthwise_conv_with_fake_quant(layer, inputs, num_bits: int | None):
if num_bits is None:
return layer(inputs)
if not layer.built:
layer.build(inputs.shape)
kernel32 = tf.cast(layer.kernel, tf.float32)
input_channels = int(kernel32.shape[-2])
output_channels = input_channels * layer.depth_multiplier
flattened = tf.reshape(
kernel32, [layer.kernel_size[0], layer.kernel_size[1], output_channels]
)
quantized = _fake_quantize_kernel_per_output_channel(flattened, num_bits)
kernel = tf.cast(tf.reshape(quantized, tf.shape(kernel32)), layer.compute_dtype)
inputs = tf.cast(inputs, layer.compute_dtype)
output = tf.keras.ops.depthwise_conv(
inputs,
kernel,
strides=layer.strides,
padding=layer.padding,
dilation_rate=layer.dilation_rate,
data_format=layer.data_format,
)
if layer.use_bias:
output = tf.nn.bias_add(
output, tf.cast(layer.bias, layer.compute_dtype), data_format="NHWC"
)
if layer.activation is not None:
output = layer.activation(output)
return output
def _separable_conv_with_fake_quant(layer, inputs, num_bits: int | None):
if num_bits is None:
return layer(inputs)
if not layer.built:
layer.build(inputs.shape)
depthwise32 = tf.cast(layer.depthwise_kernel, tf.float32)
output_channels = int(depthwise32.shape[-2]) * layer.depth_multiplier
flattened = tf.reshape(
depthwise32,
[layer.kernel_size[0], layer.kernel_size[1], output_channels],
)
depthwise = _fake_quantize_kernel_per_output_channel(flattened, num_bits)
depthwise = tf.cast(tf.reshape(depthwise, tf.shape(depthwise32)), layer.compute_dtype)
pointwise = _fake_quantize_kernel_per_output_channel(
layer.pointwise_kernel, num_bits
)
pointwise = tf.cast(pointwise, layer.compute_dtype)
inputs = tf.cast(inputs, layer.compute_dtype)
output = tf.keras.ops.separable_conv(
inputs,
depthwise,
pointwise,
strides=layer.strides,
padding=layer.padding,
data_format=layer.data_format,
dilation_rate=layer.dilation_rate,
)
if layer.use_bias:
output = tf.nn.bias_add(
output, tf.cast(layer.bias, layer.compute_dtype), data_format="NHWC"
)
if layer.activation is not None:
output = layer.activation(output)
return output
def _dense_with_fake_quant(layer, inputs, num_bits: int | None):
if num_bits is None:
return layer(inputs)
if not layer.built:
layer.build(inputs.shape)
kernel = _fake_quantize_kernel_per_output_channel(layer.kernel, num_bits)
output = tf.keras.ops.matmul(
tf.cast(inputs, layer.compute_dtype), tf.cast(kernel, layer.compute_dtype)
)
if layer.bias is not None:
output = tf.keras.ops.add(output, tf.cast(layer.bias, layer.compute_dtype))
if layer.activation is not None:
output = layer.activation(output)
return output
def _einsum_dense_with_fake_quant(
layer, inputs, num_bits: int, *, flatten_output_dimensions: bool = False
):
if not layer.built:
layer.build(inputs.shape)
kernel32 = tf.cast(layer.kernel, tf.float32)
if flatten_output_dimensions:
original_shape = tf.shape(kernel32)
flat_shape = tf.stack([original_shape[0], -1])
flattened = tf.reshape(kernel32, flat_shape)
quantized = _fake_quantize_kernel_per_output_channel(flattened, num_bits)
kernel = tf.reshape(quantized, original_shape)
else:
kernel = _fake_quantize_kernel_per_output_channel(kernel32, num_bits)
output = tf.keras.ops.einsum(
layer.equation,
tf.cast(inputs, layer.compute_dtype),
tf.cast(kernel, layer.compute_dtype),
)
if layer.bias is not None:
output = tf.keras.ops.add(output, tf.cast(layer.bias, layer.compute_dtype))
if layer.activation is not None:
output = layer.activation(output)
return output
def _kernel_parameter_count(layer) -> int:
count = 0
for name in ("kernel", "depthwise_kernel", "pointwise_kernel"):
kernel = getattr(layer, name, None)
if kernel is not None:
count += int(tf.TensorShape(kernel.shape).num_elements() or 0)
return count
@tf.keras.utils.register_keras_serializable(package="imagenet_research")
class DCNv4Reference(tf.keras.layers.Layer):
"""Differentiable TensorFlow reference for grouped DCNv4 aggregation.
This implements the published operator's value projection, learned offsets
and unnormalized aggregation weights, bilinear sampling, and output
projection. It is a portable reference implementation; it does not include
OpenGVLab's fused CUDA kernel or its memory-access optimizations.
"""
def __init__(
self,
channels: int,
groups: int = 8,
kernel_size: int = 3,
offset_scale: float = 1.0,
main_weight_bits: int | None = None,
**kwargs,
):
super().__init__(**kwargs)
if channels % groups:
raise ValueError(f"channels={channels} must be divisible by groups={groups}")
if kernel_size < 1 or kernel_size % 2 == 0:
raise ValueError("kernel_size must be a positive odd number")
if main_weight_bits not in (None, 4, 8):
raise ValueError("main_weight_bits must be None, 4, or 8")
self.channels = channels
self.groups = groups
self.kernel_size = kernel_size
self.offset_scale = float(offset_scale)
self.main_weight_bits = main_weight_bits
self.points = kernel_size * kernel_size
self.value_proj = tf.keras.layers.Conv2D(
channels, 1, use_bias=True, kernel_initializer="glorot_uniform", name="value_proj"
)
self.offset_mask_projection = tf.keras.layers.Conv2D(
groups * self.points * 3,
1,
use_bias=True,
kernel_initializer="zeros",
bias_initializer="zeros",
dtype="float32",
name="offset_mask_projection",
)
self.output_proj = tf.keras.layers.Conv2D(
channels, 1, use_bias=True, kernel_initializer="glorot_uniform", name="output_proj"
)
def build(self, input_shape):
if input_shape[-1] is not None and int(input_shape[-1]) != self.channels:
raise ValueError(
f"DCNv4 channels={self.channels} does not match input channels={input_shape[-1]}"
)
self.value_proj.build(input_shape)
self.offset_mask_projection.build(input_shape)
self.output_proj.build(input_shape)
super().build(input_shape)
def _aggregate(self, values, offsets, aggregation_weights):
shape = tf.shape(values)
batch, height, width = shape[0], shape[1], shape[2]
channels_per_group = self.channels // self.groups
padding = self.kernel_size // 2
# OpenGVLab enumerates sampling points with width outermost; keep this
# order aligned with the predictor's per-point offset and weight slots.
kernel_offsets = tf.constant(
[
(row - padding, column - padding)
for column in range(self.kernel_size)
for row in range(self.kernel_size)
],
dtype=tf.float32,
)
grid_y = tf.cast(tf.range(height), tf.float32)[None, :, None, None, None]
grid_x = tf.cast(tf.range(width), tf.float32)[None, None, :, None, None]
point_y = kernel_offsets[:, 0][None, None, None, None, :]
point_x = kernel_offsets[:, 1][None, None, None, None, :]
# The official DCNv4 predictor stores each displacement as (x, y),
# then scales both the regular kernel lattice and learned displacement.
sample_y = (
grid_y + point_y * self.offset_scale + offsets[..., 1] * self.offset_scale
)
sample_x = (
grid_x + point_x * self.offset_scale + offsets[..., 0] * self.offset_scale
)
y0_float = tf.floor(sample_y)
x0_float = tf.floor(sample_x)
y0 = tf.cast(y0_float, tf.int32)
x0 = tf.cast(x0_float, tf.int32)
y1 = y0 + 1
x1 = x0 + 1
fraction_y = sample_y - y0_float
fraction_x = sample_x - x0_float
grouped_values = tf.reshape(
values,
[batch, height, width, self.groups, channels_per_group],
)
grouped_values = tf.transpose(grouped_values, [0, 3, 1, 2, 4])
grouped_values = tf.reshape(
grouped_values,
[batch * self.groups, height * width, channels_per_group],
)
def gather_neighbour(y_index, x_index):
valid = (
(y_index >= 0)
& (y_index < height)
& (x_index >= 0)
& (x_index < width)
)
flat_index = tf.clip_by_value(y_index, 0, height - 1) * width
flat_index += tf.clip_by_value(x_index, 0, width - 1)
flat_index = tf.transpose(flat_index, [0, 3, 1, 2, 4])
valid = tf.transpose(valid, [0, 3, 1, 2, 4])
flat_index = tf.reshape(flat_index, [batch * self.groups, height * width * self.points])
valid = tf.reshape(valid, [batch * self.groups, height * width * self.points])
samples = tf.gather(
grouped_values, flat_index, axis=1, batch_dims=1
)
samples = tf.reshape(
samples,
[batch, self.groups, height * width, self.points, channels_per_group],
)
valid = tf.reshape(
valid, [batch, self.groups, height * width, self.points, 1]
)
return samples * tf.cast(valid, samples.dtype)
fraction_y = tf.transpose(fraction_y, [0, 3, 1, 2, 4])
fraction_x = tf.transpose(fraction_x, [0, 3, 1, 2, 4])
fraction_y = tf.reshape(fraction_y, [batch, self.groups, height * width, self.points])
fraction_x = tf.reshape(fraction_x, [batch, self.groups, height * width, self.points])
weights = tf.transpose(aggregation_weights, [0, 3, 1, 2, 4])
weights = tf.reshape(weights, [batch, self.groups, height * width, self.points])
top_left = gather_neighbour(y0, x0)
top_right = gather_neighbour(y0, x1)
bottom_left = gather_neighbour(y1, x0)
bottom_right = gather_neighbour(y1, x1)
sampled = (
top_left * ((1.0 - fraction_y) * (1.0 - fraction_x))[..., None]
+ top_right * ((1.0 - fraction_y) * fraction_x)[..., None]
+ bottom_left * (fraction_y * (1.0 - fraction_x))[..., None]
+ bottom_right * (fraction_y * fraction_x)[..., None]
)
aggregated = tf.reduce_sum(sampled * weights[..., None], axis=3)
aggregated = tf.transpose(aggregated, [0, 2, 1, 3])
return tf.reshape(aggregated, [batch, height, width, self.channels])
def call(self, inputs):
original_dtype = inputs.dtype
inputs32 = tf.cast(inputs, tf.float32)
values = _conv2d_with_fake_quant(
self.value_proj, inputs32, self.main_weight_bits
)
offset_mask = self.offset_mask_projection(inputs32)
shape = tf.shape(offset_mask)
offset_mask = tf.reshape(
offset_mask,
[shape[0], shape[1], shape[2], self.groups, self.points * 3],
)
# DCNv4 packs each group's 2*K x/y offsets first, followed by K
# aggregation weights. Keep this ordering aligned with the official
# kernel's group stride and its p_offset_ptr + 2*K weight offset.
offsets = tf.reshape(
offset_mask[..., : self.points * 2],
[shape[0], shape[1], shape[2], self.groups, self.points, 2],
)
aggregation_weights = offset_mask[..., self.points * 2 :]
output = self._aggregate(
tf.cast(values, tf.float32), offsets, aggregation_weights
)
output = _conv2d_with_fake_quant(
self.output_proj, output, self.main_weight_bits
)
return tf.cast(output, original_dtype)
def compute_output_shape(self, input_shape):
return input_shape
def get_config(self):
config = super().get_config()
config.update(
{
"channels": self.channels,
"groups": self.groups,
"kernel_size": self.kernel_size,
"offset_scale": self.offset_scale,
"main_weight_bits": self.main_weight_bits,
}
)
return config
def _mbconv(
x,
filters_in,
filters_out,
kernel,
stride,
expansion,
se_ratio,
drop_rate,
name,
dcnv4_spatial: bool = False,
dcnv4_main_weight_bits: int | None = None,
):
residual = x
expanded_filters = filters_in * expansion
if expansion != 1:
x = _conv_bn_relu6(
x, expanded_filters, 1, 1, f"{name}_expand", activation=True
)
if dcnv4_spatial:
if stride != 1 or kernel != 3:
raise ValueError("The DCNv4 reference path currently supports stride-1 3x3 blocks")
x = DCNv4Reference(
channels=expanded_filters,
groups=8,
kernel_size=3,
main_weight_bits=dcnv4_main_weight_bits,
name=f"{name}_dcnv4",
)(x)
else:
x = tf.keras.layers.DepthwiseConv2D(
kernel,
strides=stride,
padding="same",
use_bias=False,
depthwise_initializer="he_normal",
name=f"{name}_depthwise",
)(x)
x = tf.keras.layers.BatchNormalization(
momentum=0.99, epsilon=1e-3, name=f"{name}_depthwise_bn"
)(x)
x = tf.keras.layers.ReLU(max_value=6.0, name=f"{name}_depthwise_relu6")(x)
# The SE gate acts on the expanded tensor, but its squeeze width is based
# on the MBConv input channels.
squeeze_filters = max(1, int(filters_in * se_ratio))
se = tf.keras.layers.GlobalAveragePooling2D(keepdims=True, name=f"{name}_se_pool")(x)
se = tf.keras.layers.Conv2D(
squeeze_filters,
1,
activation="relu",
kernel_initializer="he_normal",
name=f"{name}_se_reduce",
)(se)
se = tf.keras.layers.Conv2D(
expanded_filters,
1,
activation="sigmoid",
kernel_initializer="glorot_uniform",
name=f"{name}_se_expand",
)(se)
x = tf.keras.layers.Multiply(name=f"{name}_se_scale")([x, se])
x = tf.keras.layers.Conv2D(
filters_out,
1,
padding="same",
use_bias=False,
kernel_initializer="he_normal",
name=f"{name}_project_conv",
)(x)
x = tf.keras.layers.BatchNormalization(
momentum=0.99, epsilon=1e-3, name=f"{name}_project_bn"
)(x)
if stride == 1 and filters_in == filters_out:
if drop_rate > 0:
x = tf.keras.layers.Dropout(
drop_rate,
noise_shape=(None, 1, 1, 1),
name=f"{name}_drop_path",
)(x)
x = tf.keras.layers.Add(name=f"{name}_add")([residual, x])
return x
@tf.keras.utils.register_keras_serializable(package="imagenet_research")
class SpatialTransformerBlock(tf.keras.layers.Layer):
"""Global spatial MHSA with depthwise positional mixing and an MLP."""
def __init__(self, channels: int, heads: int = 4, mlp_ratio: int = 4,
drop_rate: float = 0.0,
weight_fake_quant_bits: int | None = None, **kwargs):
super().__init__(**kwargs)
if channels % heads:
raise ValueError(f"channels={channels} must be divisible by heads={heads}")
if weight_fake_quant_bits not in (None, 4, 8):
raise ValueError("weight_fake_quant_bits must be None, 4, or 8")
self.channels = channels
self.heads = heads
self.mlp_ratio = mlp_ratio
self.drop_rate = drop_rate
self.weight_fake_quant_bits = weight_fake_quant_bits
self.position_conv = tf.keras.layers.DepthwiseConv2D(
3, padding="same", name="position_conv"
)
self.attention_norm = tf.keras.layers.LayerNormalization(
epsilon=1e-6, name="attention_norm"
)
self.attention = tf.keras.layers.MultiHeadAttention(
num_heads=heads,
key_dim=channels // heads,
dropout=drop_rate,
name="mhsa",
)
self.attention_dropout = tf.keras.layers.Dropout(drop_rate)
self.mlp_norm = tf.keras.layers.LayerNormalization(epsilon=1e-6, name="mlp_norm")
self.mlp_expand = tf.keras.layers.Dense(
channels * mlp_ratio, activation=tf.nn.relu6, name="mlp_expand"
)
self.mlp_dropout = tf.keras.layers.Dropout(drop_rate)
self.mlp_project = tf.keras.layers.Dense(channels, name="mlp_project")
def call(self, inputs, training=None):
bits = self.weight_fake_quant_bits
position = _depthwise_conv_with_fake_quant(self.position_conv, inputs, bits)
x = tf.math.add(inputs, position, name="position_residual_add")
spatial_shape = tf.shape(x)
tokens = tf.reshape(x, [spatial_shape[0], -1, self.channels])
attention_input = self.attention_norm(tokens)
if bits is None:
attended = self.attention(
attention_input, attention_input, training=training
)
else:
if not self.attention.built:
self.attention.build(attention_input.shape, attention_input.shape)
query = _einsum_dense_with_fake_quant(
self.attention._query_dense,
attention_input,
bits,
flatten_output_dimensions=True,
)
key = _einsum_dense_with_fake_quant(
self.attention._key_dense,
attention_input,
bits,
flatten_output_dimensions=True,
)
value = _einsum_dense_with_fake_quant(
self.attention._value_dense,
attention_input,
bits,
flatten_output_dimensions=True,
)
attended_heads, _ = self.attention._compute_attention(
query, key, value, None, training, False
)
attended = _einsum_dense_with_fake_quant(
self.attention._output_dense, attended_heads, bits
)
attended = self.attention_dropout(attended, training=training)
tokens = tf.math.add(tokens, attended, name="attention_residual_add")
mlp = self.mlp_norm(tokens)
mlp = _dense_with_fake_quant(self.mlp_expand, mlp, bits)
mlp = self.mlp_dropout(mlp, training=training)
tokens = tf.math.add(
tokens, _dense_with_fake_quant(self.mlp_project, mlp, bits),
name="mlp_residual_add"
)
output = tf.reshape(tokens, spatial_shape)
return tf.ensure_shape(output, inputs.shape)
def fake_quantized_kernel_parameters(self) -> int:
if self.weight_fake_quant_bits is None:
return 0
count = _kernel_parameter_count(self.position_conv)
if self.attention.built:
count += sum(
_kernel_parameter_count(projection)
for projection in (
self.attention._query_dense,
self.attention._key_dense,
self.attention._value_dense,
self.attention._output_dense,
)
)
count += _kernel_parameter_count(self.mlp_expand)
count += _kernel_parameter_count(self.mlp_project)
return count
def get_config(self):
config = super().get_config()
config.update(
{
"channels": self.channels,
"heads": self.heads,
"mlp_ratio": self.mlp_ratio,
"drop_rate": self.drop_rate,
"weight_fake_quant_bits": self.weight_fake_quant_bits,
}
)
return config
def _backbone(
variant_name: str,
hybrid: bool = False,
dcnv4: bool = False,
dcnv4_main_weight_bits: int | None = None,
):
if variant_name not in VARIANTS:
raise ValueError(f"Unknown EfficientNet scale {variant_name!r}; use {tuple(VARIANTS)}")
variant = VARIANTS[variant_name]
inputs = tf.keras.Input(
shape=(variant.image_size, variant.image_size, 3), name="image"
)
stem_filters = round_filters(32, variant.width)
x = _conv_bn_relu6(inputs, stem_filters, 3, 2, "stem")
repeats_by_stage = [round_repeats(spec[3], variant.depth) for spec in STAGE_SPECS]
total_blocks = sum(repeats_by_stage)
block_index = 0
feature_p3 = feature_p4 = feature_p5 = None
filters_in = stem_filters
for stage_index, (kernel, expansion, base_filters, _, first_stride) in enumerate(
STAGE_SPECS
):
filters_out = round_filters(base_filters, variant.width)
repeats = repeats_by_stage[stage_index]
if hybrid and stage_index == 5:
x = _conv_bn_relu6(
x, filters_out, 3, first_stride, "hybrid_stage6_patch"
)
for block in range(repeats):
drop_rate = 0.1 * block_index / max(total_blocks - 1, 1)
x = SpatialTransformerBlock(
filters_out,
heads=4,
mlp_ratio=4,
drop_rate=drop_rate,
name=f"hybrid_stage6_block{block + 1}",
)(x)
block_index += 1
elif hybrid and stage_index == 6:
x = _conv_bn_relu6(
x, filters_out, 1, first_stride, "hybrid_stage7_projection"
)
for block in range(repeats):
drop_rate = 0.1 * block_index / max(total_blocks - 1, 1)
x = SpatialTransformerBlock(
filters_out,
heads=4,
mlp_ratio=4,
drop_rate=drop_rate,
name=f"hybrid_stage7_block{block + 1}",
)(x)
block_index += 1
else:
for block in range(repeats):
stride = first_stride if block == 0 else 1
drop_rate = 0.1 * block_index / max(total_blocks - 1, 1)
x = _mbconv(
x,
filters_in=filters_in,
filters_out=filters_out,
kernel=kernel,
stride=stride,
expansion=expansion,
se_ratio=0.25,
drop_rate=drop_rate,
name=f"stage{stage_index + 1}_block{block + 1}",
dcnv4_spatial=dcnv4 and stage_index == 6,
dcnv4_main_weight_bits=dcnv4_main_weight_bits,
)
filters_in = filters_out
block_index += 1
if stage_index == 2:
feature_p3 = x
elif stage_index == 4:
feature_p4 = x
elif stage_index == 6:
feature_p5 = x
head_filters = round_filters(1280, variant.width)
head = _conv_bn_relu6(x, head_filters, 1, 1, "head")
return inputs, head, (feature_p3, feature_p4, feature_p5), variant
def build_efficientnet(variant: str, num_classes: int = 1_000):
inputs, features, _, scale = _backbone(variant)
x = tf.keras.layers.GlobalAveragePooling2D(name="global_pool")(features)
x = tf.keras.layers.Dropout(scale.dropout, name="classifier_dropout")(x)
logits = tf.keras.layers.Dense(num_classes, dtype="float32", name="logits")(x)
return tf.keras.Model(inputs, logits, name=f"efficientnet_{variant}_relu6")
def build_efficientnet_dcnv4(
variant: str,
num_classes: int = 1_000,
main_weight_bits: int | None = None,
):
inputs, features, _, scale = _backbone(
variant, dcnv4=True, dcnv4_main_weight_bits=main_weight_bits
)
x = tf.keras.layers.GlobalAveragePooling2D(name="global_pool")(features)
x = tf.keras.layers.Dropout(scale.dropout, name="classifier_dropout")(x)
logits = tf.keras.layers.Dense(num_classes, dtype="float32", name="logits")(x)
return tf.keras.Model(
inputs, logits, name=f"efficientnet_{variant}_dcnv4_relu6"
)
@tf.keras.utils.register_keras_serializable(package="imagenet_research")
class FastNormalizedFusion(tf.keras.layers.Layer):
"""Learn nonnegative, normalized weights for a small set of feature maps."""
def __init__(self, inputs_count: int, epsilon: float = 1e-4, **kwargs):
super().__init__(**kwargs)
self.inputs_count = inputs_count
self.epsilon = epsilon
def build(self, input_shape):
self.edge_weights = self.add_weight(
name="edge_weights",
shape=(self.inputs_count,),
initializer="ones",
trainable=True,
)
super().build(input_shape)
def call(self, inputs):
weights = tf.nn.relu(self.edge_weights)
weights = weights / (tf.reduce_sum(weights) + self.epsilon)
return tf.add_n([weights[i] * value for i, value in enumerate(inputs)])
def get_config(self):
config = super().get_config()
config.update({"inputs_count": self.inputs_count, "epsilon": self.epsilon})
return config
@tf.keras.utils.register_keras_serializable(package="imagenet_research")
class BiFPNLayer(tf.keras.layers.Layer):
def __init__(
self,
channels: int = 160,
attention: bool = False,
weight_fake_quant_bits: int | None = None,
**kwargs,
):
super().__init__(**kwargs)
if weight_fake_quant_bits not in (None, 4, 8):
raise ValueError("weight_fake_quant_bits must be None, 4, or 8")
self.channels = channels
self.attention_enabled = attention
self.weight_fake_quant_bits = weight_fake_quant_bits
self.projections = [
tf.keras.layers.Conv2D(channels, 1, padding="same", use_bias=False, name=f"input_proj_{i}")
for i in range(3)
]
self.projection_norms = [
tf.keras.layers.BatchNormalization(momentum=0.99, epsilon=1e-3, name=f"input_bn_{i}")
for i in range(3)
]
self.td4_fusion = FastNormalizedFusion(2, name="td4_fusion")
self.td3_fusion = FastNormalizedFusion(2, name="td3_fusion")
self.out4_fusion = FastNormalizedFusion(3, name="out4_fusion")
self.out5_fusion = FastNormalizedFusion(2, name="out5_fusion")
self.convs = [
tf.keras.layers.SeparableConv2D(
channels, 3, padding="same", use_bias=False, name=f"node_conv_{i}"
)
for i in range(4)
]
self.norms = [
tf.keras.layers.BatchNormalization(momentum=0.99, epsilon=1e-3, name=f"node_bn_{i}")
for i in range(4)
]
self.down3 = tf.keras.layers.SeparableConv2D(
channels, 3, strides=2, padding="same", use_bias=False, name="downsample_p3"
)
self.down4 = tf.keras.layers.SeparableConv2D(
channels, 3, strides=2, padding="same", use_bias=False, name="downsample_p4"
)
self.p4_attention = (
SpatialTransformerBlock(
channels,
heads=4,
mlp_ratio=2,
weight_fake_quant_bits=weight_fake_quant_bits,
name="p4_attention",
)
if attention
else None
)
self.p5_attention = (
SpatialTransformerBlock(
channels,
heads=4,
mlp_ratio=2,
weight_fake_quant_bits=weight_fake_quant_bits,
name="p5_attention",
)
if attention
else None
)
def _node(self, value, index, training):
value = _separable_conv_with_fake_quant(
self.convs[index], value, self.weight_fake_quant_bits
)
value = self.norms[index](value, training=training)
return tf.nn.relu6(value)
def call(self, inputs, training=None):
p3, p4, p5 = inputs
p3, p4, p5 = [
tf.nn.relu6(
norm(
_conv2d_with_fake_quant(
projection, feature, self.weight_fake_quant_bits
),
training=training,
)
)
for projection, norm, feature in zip(
self.projections, self.projection_norms, (p3, p4, p5)
)
]
p5_up = tf.image.resize(p5, tf.shape(p4)[1:3], method="nearest")
td4 = self._node(self.td4_fusion([p4, p5_up]), 0, training)
td4_up = tf.image.resize(td4, tf.shape(p3)[1:3], method="nearest")
td3 = self._node(self.td3_fusion([p3, td4_up]), 1, training)
out4 = self._node(
self.out4_fusion(
[
p4,
td4,
_separable_conv_with_fake_quant(
self.down3, td3, self.weight_fake_quant_bits
),
]
),
2,
training,
)
out5 = self._node(
self.out5_fusion(
[
p5,
_separable_conv_with_fake_quant(
self.down4, out4, self.weight_fake_quant_bits
),
]
),
3,
training,
)
if self.attention_enabled:
out4 = self.p4_attention(out4, training=training)
out5 = self.p5_attention(out5, training=training)
return td3, out4, out5
def fake_quantized_kernel_parameters(self) -> int:
if self.weight_fake_quant_bits is None:
return 0
count = sum(_kernel_parameter_count(layer) for layer in self.projections)
for layer in (*self.convs, self.down3, self.down4):
count += _kernel_parameter_count(layer)
if self.attention_enabled:
count += self.p4_attention.fake_quantized_kernel_parameters()
count += self.p5_attention.fake_quantized_kernel_parameters()
return count
def get_config(self):
config = super().get_config()
config.update(
{
"channels": self.channels,
"attention": self.attention_enabled,
"weight_fake_quant_bits": self.weight_fake_quant_bits,
}
)
return config
def build_efficientnet_bifpn(
variant: str,
num_classes: int = 1_000,
bifpn_repeats: int = 2,
channels: int = 160,
attention: bool = False,
):
inputs, _, features, scale = _backbone(variant)
for repeat in range(bifpn_repeats):
features = BiFPNLayer(
channels=channels,
attention=attention,
name=f"bifpn_{repeat + 1}",
)(features)
pooled = [
tf.keras.layers.GlobalAveragePooling2D(name=f"bifpn_pool_p{level}")(feature)
for level, feature in zip((3, 4, 5), features)
]
x = tf.keras.layers.Concatenate(name="bifpn_multiscale_pool")(pooled)
x = tf.keras.layers.Dropout(scale.dropout, name="classifier_dropout")(x)
logits = tf.keras.layers.Dense(num_classes, dtype="float32", name="logits")(x)
neck_name = "bifpn_attention" if attention else "bifpn"
model_name = f"efficientnet_{variant}_{neck_name}_relu6"
return tf.keras.Model(inputs, logits, name=model_name)
def build_hybrid(variant: str, num_classes: int = 1_000):
inputs, features, _, scale = _backbone(variant, hybrid=True)
x = tf.keras.layers.GlobalAveragePooling2D(name="global_pool")(features)
x = tf.keras.layers.Dropout(scale.dropout, name="classifier_dropout")(x)
logits = tf.keras.layers.Dense(num_classes, dtype="float32", name="logits")(x)
return tf.keras.Model(inputs, logits, name=f"efficientnet_{variant}_mhsa_hybrid")
def build_hybrid_bifpn(
variant: str, num_classes: int = 1_000, bifpn_repeats: int = 2, channels: int = 160
):
inputs, _, features, scale = _backbone(variant, hybrid=True)
for repeat in range(bifpn_repeats):
features = BiFPNLayer(
channels=channels,
name=f"hybrid_bifpn_neck_{repeat + 1}",
)(features)
pooled = [
tf.keras.layers.GlobalAveragePooling2D(name=f"hybrid_bifpn_pool_p{level}")(feature)
for level, feature in zip((3, 4, 5), features)
]
x = tf.keras.layers.Concatenate(name="hybrid_bifpn_multiscale_pool")(pooled)
x = tf.keras.layers.Dropout(scale.dropout, name="classifier_dropout")(x)
logits = tf.keras.layers.Dense(num_classes, dtype="float32", name="logits")(x)
return tf.keras.Model(inputs, logits, name=f"efficientnet_{variant}_mhsa_bifpn")
def build_model(model_name: str, variant: str | None = None, num_classes: int = 1_000):
"""Build one registered model by name for training and quantization tools."""
if model_name.startswith("dcnv4_"):
scale_name = variant or model_name.removeprefix("dcnv4_")
return build_efficientnet_dcnv4(scale_name, num_classes)
if model_name.startswith("hybrid_bifpn_"):
scale_name = variant or model_name.removeprefix("hybrid_bifpn_")
return build_hybrid_bifpn(scale_name, num_classes)
if model_name.startswith("hybrid_"):
scale_name = variant or model_name.removeprefix("hybrid_")
return build_hybrid(scale_name, num_classes)
if model_name.startswith("bifpn_"):
attention = model_name.startswith("bifpn_attention_")
prefix = "bifpn_attention_" if attention else "bifpn_"
scale_name = variant or model_name.removeprefix(prefix)
return build_efficientnet_bifpn(scale_name, num_classes, attention=attention)
scale_name = variant or model_name
return build_efficientnet(scale_name, num_classes)