"""Deterministic, forward-only BatchNorm recalibration for quantized models.""" from __future__ import annotations import hashlib import math import numpy as np import tensorflow as tf def _variable_path(variable) -> str: return str(getattr(variable, "path", variable.name)) def _hash_variables(variables) -> str: digest = hashlib.sha256() for variable in sorted(variables, key=_variable_path): values = np.asarray(variable.numpy()) digest.update(_variable_path(variable).encode("utf-8") + b"\0") digest.update(str(values.dtype).encode("ascii") + b"\0") digest.update(np.asarray(values.shape, dtype=" tuple[tf.keras.Model, dict]: """Re-estimate BatchNorm statistics on an exact ordered training subset. The input dataset must be deterministic, non-shuffled, and yield ``(images, labels, image_ids)`` or ``(images, labels)``. The recalibration clone disables Dropout/drop-path, resets moving statistics, freezes all non-BN weights, and calls the model with ``training=True``. Per-channel population mean and variance are accumulated across batches with parallel Welford updates, including the between-batch mean correction. This is a deterministic calibration pass on the supplied subset, not a claim that every deployment distribution is matched. """ if examples < 1: raise ValueError("examples must be positive") def clone_layer(layer): config = layer.get_config() if isinstance(layer, tf.keras.layers.Dropout): config["rate"] = 0.0 cloned = layer.__class__.from_config(config) cloned.trainable = isinstance(layer, tf.keras.layers.BatchNormalization) return cloned calibration = tf.keras.models.clone_model(model, clone_function=clone_layer) source_layers = {layer.name: layer for layer in model.layers} source_trainable = {layer.name: bool(layer.trainable) for layer in model.layers} for target_layer in calibration.layers: source_layer = source_layers.get(target_layer.name) if source_layer is None: raise ValueError(f"Calibration clone added unexpected layer {target_layer.name}") if source_layer.weights and target_layer.weights: target_layer.set_weights(source_layer.get_weights()) batch_norms = [ layer for layer in calibration.layers if isinstance(layer, tf.keras.layers.BatchNormalization) ] if not batch_norms: raise ValueError("The model has no BatchNormalization layers") original_momenta = {layer.name: float(layer.momentum) for layer in batch_norms} original_calls = {id(layer): (layer, layer.call) for layer in batch_norms} source_moving_variables = [ variable for layer in source_layers.values() if isinstance(layer, tf.keras.layers.BatchNormalization) for variable in (layer.moving_mean, layer.moving_variance) ] source_moving_sha = _hash_variables(source_moving_variables) moving_variable_ids = { id(variable) for layer in calibration.layers if isinstance(layer, tf.keras.layers.BatchNormalization) for variable in (layer.moving_mean, layer.moving_variance) } fixed_variables = [ variable for layer in calibration.layers for variable in layer.weights if id(variable) not in moving_variable_ids ] fixed_before = [np.asarray(variable.numpy()).copy() for variable in fixed_variables] batch_moments: dict[str, tuple[np.ndarray, np.ndarray, int]] = {} running_moments: dict[str, tuple[int, np.ndarray, np.ndarray]] = {} for layer in calibration.layers: layer.trainable = isinstance(layer, tf.keras.layers.BatchNormalization) # Freeze only affine/trainable variables; the BN layers remain trainable so # their moving statistics update during explicit training=True calls. for layer in batch_norms: layer.trainable = True layer.moving_mean.assign(tf.zeros_like(layer.moving_mean)) layer.moving_variance.assign(tf.ones_like(layer.moving_variance)) layer.momentum = 0.0 original_call = original_calls[id(layer)][1] def tracked_call(inputs, training=None, mask=None, _layer=layer, _original=original_call, **kwargs): if training is True: axes = tuple(range(len(inputs.shape) - 1)) mean, variance = tf.nn.moments(inputs, axes=axes) mean = np.asarray(mean.numpy(), dtype=np.float64) variance = np.asarray(variance.numpy(), dtype=np.float64) dynamic_shape = tf.shape(inputs).numpy() count = math.prod(int(dynamic_shape[axis]) for axis in axes) batch_moments[_layer.name] = (mean, variance, int(count)) return _original(inputs, training=training, mask=mask, **kwargs) layer.call = tracked_call observed = 0 identity = hashlib.sha256() last_batch_count = 0 has_ids = None try: for batch in dataset: if not isinstance(batch, (tuple, list)) or len(batch) < 1: raise ValueError("Dataset batches must contain images as their first item") images = batch[0] if isinstance(images, dict): raise ValueError("Image dictionaries are not supported by this recalibration helper") batch_count = int(tf.shape(images)[0].numpy()) if batch_count < 1: continue take = min(batch_count, examples - observed) if take <= 0: break images = images[:take] labels = batch[1][:take].numpy() if len(batch) >= 2 else None batch_has_ids = len(batch) >= 3 if has_ids is None: has_ids = batch_has_ids elif has_ids != batch_has_ids: raise ValueError("Dataset inconsistently includes image IDs") if batch_has_ids: image_ids = np.asarray(batch[2][:take].numpy()).reshape(-1) for index, image_id in enumerate(image_ids): identity.update( bytes(image_id) if isinstance(image_id, (bytes, np.bytes_)) else str(image_id).encode("utf-8") ) identity.update(b"\t") if labels is not None: identity.update(str(int(labels[index])).encode("ascii")) identity.update(b"\n") batch_moments.clear() calibration(images, training=True) for layer in batch_norms: if layer.name not in batch_moments: raise RuntimeError(f"BatchNorm layer {layer.name} was not called") batch_mean, batch_variance, batch_elements = batch_moments[layer.name] previous = running_moments.get(layer.name) if previous is None: running_moments[layer.name] = ( batch_elements, batch_mean.copy(), batch_variance.copy(), ) else: previous_elements, previous_mean, previous_variance = previous total_elements = previous_elements + batch_elements delta = batch_mean - previous_mean mean = previous_mean + delta * (batch_elements / total_elements) variance = ( previous_elements * previous_variance + batch_elements * batch_variance + np.square(delta) * (previous_elements * batch_elements / total_elements) ) / total_elements running_moments[layer.name] = (total_elements, mean, variance) _, mean, variance = running_moments[layer.name] layer.moving_mean.assign(mean.astype(np.float32)) layer.moving_variance.assign(variance.astype(np.float32)) observed += take last_batch_count = take if observed == examples: break finally: for layer, original_call in original_calls.values(): layer.call = original_call for layer in calibration.layers: if isinstance(layer, tf.keras.layers.BatchNormalization): layer.momentum = original_momenta[layer.name] layer.trainable = source_trainable[layer.name] calibration.trainable = model.trainable if observed != examples: raise RuntimeError(f"Calibration dataset ended after {observed}/{examples} examples") fixed_after = [np.asarray(variable.numpy()) for variable in fixed_variables] if any(not np.array_equal(before, after) for before, after in zip(fixed_before, fixed_after)): raise RuntimeError("A non-moving model weight changed during BN recalibration") moving_variables = [ variable for layer in batch_norms for variable in (layer.moving_mean, layer.moving_variance) ] return calibration, { "status": "complete", "examples": observed, "batch_norm_layer_count": len(batch_norms), "last_batch_examples": last_batch_count, "ordered_image_id_label_sha256": identity.hexdigest() if has_ids else None, "update": "forward-only training=True pass; exact per-layer pooled population moments with between-batch correction; trainable weights frozen; dropout/drop-path disabled", "per_layer_element_counts": { name: count for name, (count, _, _) in sorted(running_moments.items()) }, "source_moving_statistics_sha256": source_moving_sha, "recalibrated_moving_statistics_sha256": _hash_variables(moving_variables), "non_moving_weights_verified_unchanged": True, "interpretation_limit": "This estimates BatchNorm moments over the fixed calibration subset; it does not establish classifier-quality recovery or deployment-distribution matching.", }