efficientnet-b0-sub4bit-mixed-precision / code /batchnorm_recalibration.py
meatloafer's picture
Upload verified model weights, LiteRT FlatBuffers, DP profiles, code, and model card
2abfed0 verified
Raw History Blame Contribute Delete
10.4 kB
"""Deterministic, forward-only BatchNorm recalibration for quantized models."""
from __future__ import annotations
import hashlib
import math
import numpy as np
import tensorflow as tf
def _variable_path(variable) -> str:
return str(getattr(variable, "path", variable.name))
def _hash_variables(variables) -> str:
digest = hashlib.sha256()
for variable in sorted(variables, key=_variable_path):
values = np.asarray(variable.numpy())
digest.update(_variable_path(variable).encode("utf-8") + b"\0")
digest.update(str(values.dtype).encode("ascii") + b"\0")
digest.update(np.asarray(values.shape, dtype="<i8").tobytes())
digest.update(values.tobytes(order="C"))
return digest.hexdigest()
def recalibrate_batchnorm_forward_only(
model: tf.keras.Model,
dataset,
*,
examples: int = 1_000,
) -> tuple[tf.keras.Model, dict]:
"""Re-estimate BatchNorm statistics on an exact ordered training subset.
The input dataset must be deterministic, non-shuffled, and yield
``(images, labels, image_ids)`` or ``(images, labels)``. The recalibration
clone disables Dropout/drop-path, resets moving statistics, freezes all
non-BN weights, and calls the model with ``training=True``. Per-channel
population mean and variance are accumulated across batches with parallel
Welford updates, including the between-batch mean correction. This is a
deterministic calibration pass on the supplied subset, not a claim that
every deployment distribution is matched.
"""
if examples < 1:
raise ValueError("examples must be positive")
def clone_layer(layer):
config = layer.get_config()
if isinstance(layer, tf.keras.layers.Dropout):
config["rate"] = 0.0
cloned = layer.__class__.from_config(config)
cloned.trainable = isinstance(layer, tf.keras.layers.BatchNormalization)
return cloned
calibration = tf.keras.models.clone_model(model, clone_function=clone_layer)
source_layers = {layer.name: layer for layer in model.layers}
source_trainable = {layer.name: bool(layer.trainable) for layer in model.layers}
for target_layer in calibration.layers:
source_layer = source_layers.get(target_layer.name)
if source_layer is None:
raise ValueError(f"Calibration clone added unexpected layer {target_layer.name}")
if source_layer.weights and target_layer.weights:
target_layer.set_weights(source_layer.get_weights())
batch_norms = [
layer for layer in calibration.layers
if isinstance(layer, tf.keras.layers.BatchNormalization)
]
if not batch_norms:
raise ValueError("The model has no BatchNormalization layers")
original_momenta = {layer.name: float(layer.momentum) for layer in batch_norms}
original_calls = {id(layer): (layer, layer.call) for layer in batch_norms}
source_moving_variables = [
variable
for layer in source_layers.values()
if isinstance(layer, tf.keras.layers.BatchNormalization)
for variable in (layer.moving_mean, layer.moving_variance)
]
source_moving_sha = _hash_variables(source_moving_variables)
moving_variable_ids = {
id(variable)
for layer in calibration.layers
if isinstance(layer, tf.keras.layers.BatchNormalization)
for variable in (layer.moving_mean, layer.moving_variance)
}
fixed_variables = [
variable
for layer in calibration.layers
for variable in layer.weights
if id(variable) not in moving_variable_ids
]
fixed_before = [np.asarray(variable.numpy()).copy() for variable in fixed_variables]
batch_moments: dict[str, tuple[np.ndarray, np.ndarray, int]] = {}
running_moments: dict[str, tuple[int, np.ndarray, np.ndarray]] = {}
for layer in calibration.layers:
layer.trainable = isinstance(layer, tf.keras.layers.BatchNormalization)
# Freeze only affine/trainable variables; the BN layers remain trainable so
# their moving statistics update during explicit training=True calls.
for layer in batch_norms:
layer.trainable = True
layer.moving_mean.assign(tf.zeros_like(layer.moving_mean))
layer.moving_variance.assign(tf.ones_like(layer.moving_variance))
layer.momentum = 0.0
original_call = original_calls[id(layer)][1]
def tracked_call(inputs, training=None, mask=None, _layer=layer, _original=original_call, **kwargs):
if training is True:
axes = tuple(range(len(inputs.shape) - 1))
mean, variance = tf.nn.moments(inputs, axes=axes)
mean = np.asarray(mean.numpy(), dtype=np.float64)
variance = np.asarray(variance.numpy(), dtype=np.float64)
dynamic_shape = tf.shape(inputs).numpy()
count = math.prod(int(dynamic_shape[axis]) for axis in axes)
batch_moments[_layer.name] = (mean, variance, int(count))
return _original(inputs, training=training, mask=mask, **kwargs)
layer.call = tracked_call
observed = 0
identity = hashlib.sha256()
last_batch_count = 0
has_ids = None
try:
for batch in dataset:
if not isinstance(batch, (tuple, list)) or len(batch) < 1:
raise ValueError("Dataset batches must contain images as their first item")
images = batch[0]
if isinstance(images, dict):
raise ValueError("Image dictionaries are not supported by this recalibration helper")
batch_count = int(tf.shape(images)[0].numpy())
if batch_count < 1:
continue
take = min(batch_count, examples - observed)
if take <= 0:
break
images = images[:take]
labels = batch[1][:take].numpy() if len(batch) >= 2 else None
batch_has_ids = len(batch) >= 3
if has_ids is None:
has_ids = batch_has_ids
elif has_ids != batch_has_ids:
raise ValueError("Dataset inconsistently includes image IDs")
if batch_has_ids:
image_ids = np.asarray(batch[2][:take].numpy()).reshape(-1)
for index, image_id in enumerate(image_ids):
identity.update(
bytes(image_id)
if isinstance(image_id, (bytes, np.bytes_))
else str(image_id).encode("utf-8")
)
identity.update(b"\t")
if labels is not None:
identity.update(str(int(labels[index])).encode("ascii"))
identity.update(b"\n")
batch_moments.clear()
calibration(images, training=True)
for layer in batch_norms:
if layer.name not in batch_moments:
raise RuntimeError(f"BatchNorm layer {layer.name} was not called")
batch_mean, batch_variance, batch_elements = batch_moments[layer.name]
previous = running_moments.get(layer.name)
if previous is None:
running_moments[layer.name] = (
batch_elements,
batch_mean.copy(),
batch_variance.copy(),
)
else:
previous_elements, previous_mean, previous_variance = previous
total_elements = previous_elements + batch_elements
delta = batch_mean - previous_mean
mean = previous_mean + delta * (batch_elements / total_elements)
variance = (
previous_elements * previous_variance
+ batch_elements * batch_variance
+ np.square(delta)
* (previous_elements * batch_elements / total_elements)
) / total_elements
running_moments[layer.name] = (total_elements, mean, variance)
_, mean, variance = running_moments[layer.name]
layer.moving_mean.assign(mean.astype(np.float32))
layer.moving_variance.assign(variance.astype(np.float32))
observed += take
last_batch_count = take
if observed == examples:
break
finally:
for layer, original_call in original_calls.values():
layer.call = original_call
for layer in calibration.layers:
if isinstance(layer, tf.keras.layers.BatchNormalization):
layer.momentum = original_momenta[layer.name]
layer.trainable = source_trainable[layer.name]
calibration.trainable = model.trainable
if observed != examples:
raise RuntimeError(f"Calibration dataset ended after {observed}/{examples} examples")
fixed_after = [np.asarray(variable.numpy()) for variable in fixed_variables]
if any(not np.array_equal(before, after) for before, after in zip(fixed_before, fixed_after)):
raise RuntimeError("A non-moving model weight changed during BN recalibration")
moving_variables = [
variable for layer in batch_norms
for variable in (layer.moving_mean, layer.moving_variance)
]
return calibration, {
"status": "complete",
"examples": observed,
"batch_norm_layer_count": len(batch_norms),
"last_batch_examples": last_batch_count,
"ordered_image_id_label_sha256": identity.hexdigest() if has_ids else None,
"update": "forward-only training=True pass; exact per-layer pooled population moments with between-batch correction; trainable weights frozen; dropout/drop-path disabled",
"per_layer_element_counts": {
name: count for name, (count, _, _) in sorted(running_moments.items())
},
"source_moving_statistics_sha256": source_moving_sha,
"recalibrated_moving_statistics_sha256": _hash_variables(moving_variables),
"non_moving_weights_verified_unchanged": True,
"interpretation_limit": "This estimates BatchNorm moments over the fixed calibration subset; it does not establish classifier-quality recovery or deployment-distribution matching.",
}