efficientnet-b0-stage-attention / code /attention_conv_model.py
meatloafer's picture
Upload verified model weights, LiteRT FlatBuffers, DP profiles, code, and model card
587d427 verified
Raw History Blame Contribute Delete
14.4 kB
"""Sparse multiscale attention adapters on an EfficientNet convolutional trunk.
This builder is intentionally kept separate from ``models.py``. It preserves
every MBConv stage and
adds three residual attention adapters: windowed attention at P3 and global
attention at P4 and P5. Zero-initialized LayerScale gates make the inserted
blocks an identity at initialization, so a transferred EfficientNet checkpoint
starts with unchanged logits.
The strict training launcher supports the ``attention_conv`` architecture
name. A run still requires a committed source-pinned registration and queue
admission after the existing priority sequence.
"""
from __future__ import annotations
import tensorflow as tf
from imagenet_research.models import (
STAGE_SPECS,
VARIANTS,
_conv_bn_relu6,
_mbconv,
round_filters,
round_repeats,
)
ATTENTION_STAGE_INDICES = (2, 4, 6)
EXPECTED_ATTENTION_LAYER_NAMES = (
"attnconv_stage3",
"attnconv_stage5",
"attnconv_stage7",
)
P3_WINDOW_SIZE = 7
ATTENTION_HEADS = 4
ATTENTION_MLP_RATIO = 2
@tf.keras.utils.register_keras_serializable(package="imagenet_research")
class ConvolutionalAttentionAdapter(tf.keras.layers.Layer):
"""Residual spatial attention with convolutional positional mixing.
``window_size=None`` uses global attention. A positive window size
partitions a feature map into non-overlapping windows, padding only the
bottom and right edges when needed. The depthwise positional convolution
is applied before partitioning, so nearby windows still receive local
convolutional context.
The attention and MLP branches have independent per-channel LayerScale
gates initialized to zero. At the initial checkpoint, the adapter is
exactly the identity; training can first open the gates and then update
each branch.
"""
def __init__(
self,
channels: int,
heads: int = ATTENTION_HEADS,
mlp_ratio: int = ATTENTION_MLP_RATIO,
window_size: int | None = None,
**kwargs,
):
super().__init__(**kwargs)
if channels < 1 or heads < 1 or channels % heads:
raise ValueError("channels must be positive and divisible by heads")
if mlp_ratio < 1:
raise ValueError("mlp_ratio must be positive")
if window_size is not None and window_size < 1:
raise ValueError("window_size must be positive or None")
self.channels = int(channels)
self.heads = int(heads)
self.mlp_ratio = int(mlp_ratio)
self.window_size = None if window_size is None else int(window_size)
self.position_conv = tf.keras.layers.DepthwiseConv2D(
3,
padding="same",
depthwise_initializer="he_normal",
name="position_conv",
)
self.attention_norm = tf.keras.layers.LayerNormalization(
epsilon=1e-6, name="attention_norm"
)
self.attention = tf.keras.layers.MultiHeadAttention(
num_heads=self.heads,
key_dim=self.channels // self.heads,
output_shape=self.channels,
name="mhsa",
)
self.mlp_norm = tf.keras.layers.LayerNormalization(
epsilon=1e-6, name="mlp_norm"
)
self.mlp_expand = tf.keras.layers.Dense(
self.channels * self.mlp_ratio,
activation=tf.nn.relu6,
name="mlp_expand",
)
self.mlp_project = tf.keras.layers.Dense(
self.channels, name="mlp_project"
)
def build(self, input_shape):
if input_shape[-1] is not None and int(input_shape[-1]) != self.channels:
raise ValueError(
f"channels={self.channels} does not match input channels={input_shape[-1]}"
)
self.attention_scale = self.add_weight(
name="attention_scale",
shape=(self.channels,),
initializer="zeros",
trainable=True,
)
self.mlp_scale = self.add_weight(
name="mlp_scale",
shape=(self.channels,),
initializer="zeros",
trainable=True,
)
super().build(input_shape)
def _window_attention(self, tokens, training):
window = self.window_size
shape = tf.shape(tokens)
batch, height, width = shape[0], shape[1], shape[2]
pad_height = tf.math.floormod(-height, window)
pad_width = tf.math.floormod(-width, window)
paddings = tf.stack(
[
tf.constant([0, 0], dtype=tf.int32),
tf.stack([tf.constant(0, tf.int32), pad_height]),
tf.stack([tf.constant(0, tf.int32), pad_width]),
tf.constant([0, 0], dtype=tf.int32),
]
)
padded = tf.pad(tokens, paddings)
padded_shape = tf.shape(padded)
windows_h = padded_shape[1] // window
windows_w = padded_shape[2] // window
windows = tf.reshape(
padded,
[batch, windows_h, window, windows_w, window, self.channels],
)
windows = tf.transpose(windows, [0, 1, 3, 2, 4, 5])
windows = tf.reshape(windows, [-1, window * window, self.channels])
windows = self.attention_norm(windows)
attended = self.attention(
windows, windows, training=training
)
attended = tf.reshape(
attended,
[batch, windows_h, windows_w, window, window, self.channels],
)
attended = tf.transpose(attended, [0, 1, 3, 2, 4, 5])
attended = tf.reshape(
attended,
[batch, padded_shape[1], padded_shape[2], self.channels],
)
return attended[:, :height, :width, :]
def _global_attention(self, tokens, training):
shape = tf.shape(tokens)
flattened = tf.reshape(tokens, [shape[0], shape[1] * shape[2], self.channels])
normalized = self.attention_norm(flattened)
attended = self.attention(
normalized, normalized, training=training
)
return tf.reshape(attended, shape)
def call(self, inputs, training=None):
inputs = tf.cast(inputs, self.compute_dtype)
position = self.position_conv(inputs)
positioned = inputs + tf.cast(position, inputs.dtype)
if self.window_size is None:
attended = self._global_attention(positioned, training)
else:
attended = self._window_attention(positioned, training)
attended = tf.cast(attended, inputs.dtype)
x = inputs + tf.cast(self.attention_scale, inputs.dtype) * attended
shape = tf.shape(x)
tokens = tf.reshape(x, [shape[0], shape[1] * shape[2], self.channels])
mlp = self.mlp_norm(tokens)
mlp = self.mlp_expand(mlp)
mlp = self.mlp_project(mlp)
tokens = tokens + tf.cast(self.mlp_scale, tokens.dtype) * tf.cast(
mlp, tokens.dtype
)
output = tf.reshape(tokens, shape)
return tf.ensure_shape(output, inputs.shape)
def get_config(self):
config = super().get_config()
config.update(
{
"channels": self.channels,
"heads": self.heads,
"mlp_ratio": self.mlp_ratio,
"window_size": self.window_size,
}
)
return config
def build_efficientnet_attention_conv(
variant: str,
num_classes: int = 1_000,
) -> tf.keras.Model:
"""Build EfficientNet with P3-windowed and P4/P5-global attention adapters."""
if variant not in VARIANTS:
raise ValueError(f"Unknown EfficientNet scale {variant!r}; use {tuple(VARIANTS)}")
scale = VARIANTS[variant]
inputs = tf.keras.Input(
shape=(scale.image_size, scale.image_size, 3), name="image"
)
stem_filters = round_filters(32, scale.width)
x = _conv_bn_relu6(inputs, stem_filters, 3, 2, "stem")
repeats_by_stage = [round_repeats(spec[3], scale.depth) for spec in STAGE_SPECS]
total_blocks = sum(repeats_by_stage)
filters_in = stem_filters
block_index = 0
for stage_index, (kernel, expansion, base_filters, _, first_stride) in enumerate(
STAGE_SPECS
):
filters_out = round_filters(base_filters, scale.width)
repeats = repeats_by_stage[stage_index]
for block in range(repeats):
stride = first_stride if block == 0 else 1
drop_rate = 0.1 * block_index / max(total_blocks - 1, 1)
x = _mbconv(
x,
filters_in=filters_in,
filters_out=filters_out,
kernel=kernel,
stride=stride,
expansion=expansion,
se_ratio=0.25,
drop_rate=drop_rate,
name=f"stage{stage_index + 1}_block{block + 1}",
)
filters_in = filters_out
block_index += 1
if stage_index in ATTENTION_STAGE_INDICES:
channels = filters_out
window_size = P3_WINDOW_SIZE if stage_index == 2 else None
x = ConvolutionalAttentionAdapter(
channels=channels,
heads=ATTENTION_HEADS,
mlp_ratio=ATTENTION_MLP_RATIO,
window_size=window_size,
name=f"attnconv_stage{stage_index + 1}",
)(x)
head_filters = round_filters(1280, scale.width)
x = _conv_bn_relu6(x, head_filters, 1, 1, "head")
x = tf.keras.layers.GlobalAveragePooling2D(name="global_pool")(x)
x = tf.keras.layers.Dropout(scale.dropout, name="classifier_dropout")(x)
logits = tf.keras.layers.Dense(num_classes, dtype="float32", name="logits")(x)
return tf.keras.Model(
inputs,
logits,
name=f"efficientnet_{variant}_sparse_attnconv_relu6",
)
def transfer_matching_backbone_weights(
source_model: tf.keras.Model,
target_model: tf.keras.Model,
) -> dict[str, object]:
"""Copy every exact name-and-shape match and report new target weights.
Fails closed if any weighted source layer is missing or has a different
weight layout, or if the target-only weighted layers are not exactly the
three planned attention adapters. Call this before the parity audit so a
transferred EfficientNet function is the actual audit baseline.
"""
source_layers = {layer.name: layer for layer in source_model.layers}
target_layers = {layer.name: layer for layer in target_model.layers}
copied_layers: list[str] = []
missing_layers: list[str] = []
shape_mismatches: dict[str, dict[str, list[tuple[int, ...]]]] = {}
for name, source_layer in source_layers.items():
source_weights = source_layer.get_weights()
if not source_weights:
continue
target_layer = target_layers.get(name)
if target_layer is None:
missing_layers.append(name)
continue
target_weights = target_layer.get_weights()
source_shapes = [tuple(array.shape) for array in source_weights]
target_shapes = [tuple(array.shape) for array in target_weights]
if source_shapes != target_shapes:
shape_mismatches[name] = {
"source": source_shapes,
"target": target_shapes,
}
continue
target_layer.set_weights(source_weights)
copied_layers.append(name)
source_weighted_names = {
layer.name for layer in source_model.layers if layer.get_weights()
}
target_only_weighted = sorted(
layer.name
for layer in target_model.layers
if layer.get_weights() and layer.name not in source_weighted_names
)
expected_target_only = sorted(EXPECTED_ATTENTION_LAYER_NAMES)
target_only_set_matches_expected = target_only_weighted == expected_target_only
report: dict[str, object] = {
"copied_weighted_layer_count": len(copied_layers),
"copied_weighted_layers": copied_layers,
"missing_source_layers": missing_layers,
"shape_mismatches": shape_mismatches,
"target_only_weighted_layers": target_only_weighted,
"expected_target_only_weighted_layers": expected_target_only,
"target_only_set_matches_expected": target_only_set_matches_expected,
"complete_source_transfer": (
not missing_layers
and not shape_mismatches
and target_only_set_matches_expected
),
}
if missing_layers or shape_mismatches or not target_only_set_matches_expected:
raise ValueError(f"Incomplete EfficientNet weight transfer: {report}")
return report
def audit_initial_logits_parity(
source_model: tf.keras.Model,
target_model: tf.keras.Model,
images: tf.Tensor,
*,
atol: float = 1e-6,
rtol: float = 1e-6,
) -> dict[str, object]:
"""Measure inference-logit parity on a deterministic input batch.
Use validation examples with ``training=False`` after weight transfer.
The audit returns metrics instead of silently accepting a mismatch; the
caller should record the report and require ``within_tolerance`` before
training the attention-convolution arm.
"""
images = tf.convert_to_tensor(images)
if images.shape.rank != 4:
raise ValueError("images must be a rank-4 NHWC batch")
if images.shape[0] == 0:
raise ValueError("images must contain at least one example")
reference_logits = tf.cast(source_model(images, training=False), tf.float64)
target_logits = tf.cast(target_model(images, training=False), tf.float64)
difference = tf.abs(reference_logits - target_logits)
finite = tf.reduce_all(tf.math.is_finite(difference))
within_tolerance = tf.reduce_all(
difference <= tf.cast(atol, tf.float64)
+ tf.cast(rtol, tf.float64) * tf.abs(reference_logits)
)
return {
"example_count": int(tf.shape(images)[0].numpy()),
"logit_count": int(tf.size(difference).numpy()),
"max_absolute_difference": float(tf.reduce_max(difference).numpy()),
"mean_absolute_difference": float(tf.reduce_mean(difference).numpy()),
"finite": bool(finite.numpy()),
"within_tolerance": bool(tf.logical_and(finite, within_tolerance).numpy()),
"atol": float(atol),
"rtol": float(rtol),
}