"""Boundary U-Nets, image context, axial decisions, and training losses.""" from __future__ import annotations import math import torch from torch import nn from torch.nn import functional as F class Block(nn.Module): def __init__(self, cin, cout): super().__init__() groups = min(8, cout) self.net = nn.Sequential( nn.Conv2d(cin, cout, 3, padding=1, bias=False), nn.GroupNorm(groups, cout), nn.SiLU(inplace=True), nn.Conv2d(cout, cout, 3, padding=1, bias=False), nn.GroupNorm(groups, cout), nn.SiLU(inplace=True), ) def forward(self, x): return self.net(x) class BoundaryNet(nn.Module): def __init__(self, boundary_keys, base=16, in_channels=1): super().__init__() self.boundary_keys = boundary_keys channels = [base, 2 * base, 4 * base, 8 * base, 12 * base] self.encoder = nn.ModuleList( [ Block(in_channels if i == 0 else channels[i - 1], count) for i, count in enumerate(channels) ] ) self.decoder = nn.ModuleList( [ Block(channels[i] + channels[i - 1], channels[i - 1]) for i in range(4, 0, -1) ] ) self.heads = nn.ModuleDict( {d: nn.Conv2d(base, len(keys), 1) for d, keys in boundary_keys.items()} ) def encode(self, x): """Return the bottleneck and skip features without changing state keys.""" skips = [] for i, enc in enumerate(self.encoder): if i: x = F.avg_pool2d(x, 2) x = enc(x) skips.append(x) return x, skips def decode(self, x, skips): """Reconstruct image features from a bottleneck and encoder skips.""" for dec, skip in zip(self.decoder, reversed(skips[:-1])): x = F.interpolate( x, size=skip.shape[-2:], mode="bilinear", align_corners=False ) x = dec(torch.cat([x, skip], dim=1)) return x def forward(self, x, device): x, skips = self.encode(x) x = self.decode(x, skips) return self.heads[device](x) def probability_median(probability): """Median of normalized, piecewise-uniform row bins; preserve endpoints.""" p = probability.float() height = p.shape[-2] cdf = p.cumsum(-2) index = (cdf >= 0.5).to(torch.int64).argmax(-2, keepdim=True) previous = cdf.gather(-2, (index - 1).clamp_min(0)) previous = torch.where(index == 0, torch.zeros_like(previous), previous) mass = p.gather(-2, index).clamp_min(1e-12) fraction = ((0.5 - previous) / mass).clamp(0, 1) return (index.float() - 0.5 + fraction).clamp(0, height - 1).squeeze(-2) def decode_logits(logits, local_radius=6, method="local_centroid"): """Decode axial likelihoods into resized row coordinates.""" z = logits.float() h = z.shape[-2] if method == "posterior_median": return probability_median(z.softmax(dim=-2)) if method == "quadratic_peak": if h < 3: raise ValueError("Quadratic decoding needs at least three axial rows") peak = z.argmax(dim=-2, keepdim=True) center = peak.clamp(1, h - 2) left = z.gather(-2, center - 1) middle = z.gather(-2, center) right = z.gather(-2, center + 1) curvature = left - 2 * middle + right concave = curvature < (-1e-6) denominator = torch.where(concave, curvature, -torch.ones_like(curvature)) offset = 0.5 * (left - right) / denominator fitted = torch.where(concave, center.float() + offset, peak.float()) fitted = torch.minimum( torch.maximum(fitted, peak.float() - 0.5), peak.float() + 0.5 ) return fitted.clamp(0, h - 1).squeeze(-2) if method != "local_centroid": raise ValueError("Unknown axial likelihood decoder") grid = torch.arange(h, device=z.device, dtype=torch.float32)[None, None, :, None] if local_radius: peak = z.argmax(dim=-2, keepdim=True) z = z.masked_fill((grid - peak).abs() > local_radius, -1e4) return (z.softmax(dim=-2) * grid).sum(dim=-2) def median_bin_targets(target, height): """Two row bins whose piecewise-uniform median is the specified coordinate.""" y = target.float().clamp(0, height - 1) lower = y.floor().to(torch.int64) fraction = y - lower left = torch.where( fraction <= 0.5, 0.5 / (fraction + 0.5), (1 - fraction) / (1.5 - fraction) ) weights = torch.zeros( (*y.shape[:-1], height, y.shape[-1]), device=y.device, dtype=torch.float32 ) weights.scatter_add_(-2, lower.unsqueeze(-2), left.unsqueeze(-2)) weights.scatter_add_( -2, (lower + 1).clamp_max(height - 1).unsqueeze(-2), (1 - left).unsqueeze(-2) ) return weights def boundary_loss( logits, target, valid, sigma=1.2, coordinate_weight=0.01, coordinate_loss="mean_l1", target_encoding="gaussian", ): """Gaussian axial cross entropy with per-boundary support normalization.""" h = logits.shape[-2] grid = torch.arange(h, device=logits.device, dtype=torch.float32)[ None, None, :, None ] if target_encoding == "gaussian": weights = torch.exp(-0.5 * ((grid - target[:, :, None, :]) / sigma) ** 2) weights /= weights.sum(dim=2, keepdim=True).clamp_min(1e-8) elif target_encoding == "median_bins": weights = median_bin_targets(target, h) else: raise ValueError("Unknown boundary target encoding") ce = -(weights * logits.float().log_softmax(dim=2)).sum(dim=2) probabilities = logits.float().softmax(dim=2) if coordinate_loss == "mean_l1": distance = ((probabilities * grid).sum(dim=2) - target).abs() elif coordinate_loss == "expected_abs": distance = (probabilities * (grid - target[:, :, None, :]).abs()).sum(dim=2) elif coordinate_loss == "median_l1": distance = (decode_logits(logits, method="posterior_median") - target).abs() else: raise ValueError("Unknown coordinate loss") loss = ce + coordinate_weight * distance counts = valid.sum(dim=-1) terms = (loss * valid).sum(dim=-1) / counts.clamp_min(1) return (terms * (counts > 0)).sum() / (counts > 0).sum().clamp_min(1) class AxialContext(nn.Module): def __init__(self, channels, heads=6): super().__init__() if channels % heads: raise ValueError("Axial attention channels must divide into heads") self.heads = heads self.position = nn.Conv2d(channels, channels, 3, padding=1, groups=channels) self.norm = nn.GroupNorm(8, channels) self.qkv = nn.Conv2d(channels, 3 * channels, 1, bias=False) self.projection = nn.Conv2d(channels, channels, 1) self.ff_norm = nn.GroupNorm(8, channels) self.ff = nn.Sequential( nn.Conv2d(channels, 2 * channels, 1), nn.GELU(), nn.Conv2d(2 * channels, channels, 1), ) # Own-checkpoint expansion initially leaves all original predictions intact. nn.init.zeros_(self.projection.weight) nn.init.zeros_(self.projection.bias) nn.init.zeros_(self.ff[-1].weight) nn.init.zeros_(self.ff[-1].bias) def forward(self, x): n, c, h, w = x.shape dim = c // self.heads packed = ( self.qkv(self.norm(x + self.position(x))) .reshape(n, 3, self.heads, dim, h, w) .permute(1, 0, 5, 2, 4, 3) .reshape(3, n * w, self.heads, h, dim) ) q, k, v = packed.unbind(0) context = F.scaled_dot_product_attention(q, k, v, dropout_p=0.0) context = ( context.reshape(n, w, self.heads, h, dim) .permute(0, 2, 4, 3, 1) .contiguous() .reshape(n, c, h, w) ) x = x + self.projection(context) return x + self.ff(self.ff_norm(x)) class AxialBoundaryNet(BoundaryNet): def __init__(self, boundary_keys, base=16, in_channels=1, layers=2, heads=6): super().__init__(boundary_keys, base, in_channels) if not 1 <= layers <= 4: raise ValueError("Axial context requires one to four layers") self.axial = nn.ModuleList( [AxialContext(12 * base, heads) for _ in range(layers)] ) def forward(self, x, device): x, skips = self.encode(x) for context in self.axial: x = context(x) x = self.decode(x, skips) return self.heads[device](x) class GlobalBoundaryNet(BoundaryNet): def __init__(self, boundary_keys, base=16, in_channels=1, layers=0, heads=6): super().__init__(boundary_keys, base, in_channels) if not 0 <= layers <= 4: raise ValueError("Unsupported axial context depth") self.axial = nn.ModuleList( [AxialContext(12 * base, heads) for _ in range(layers)] ) self.edge_heads = nn.ModuleDict( { d: nn.Sequential( nn.Conv1d(24 * base, 8 * base, 1), nn.GroupNorm(8, 8 * base), nn.SiLU(), nn.Conv1d(8 * base, 8 * base, 3, padding=1), nn.SiLU(), nn.Conv1d(8 * base, len(keys), 1), ) for d, keys in boundary_keys.items() } ) # Preserve the original model exactly when expanding an own checkpoint. for head in self.edge_heads.values(): nn.init.zeros_(head[-1].weight) nn.init.zeros_(head[-1].bias) def forward(self, x, device): x, skips = self.encode(x) for context in self.axial: x = context(x) summary = torch.cat([x.mean(dim=2), x.amax(dim=2)], dim=1) edge = self.edge_heads[device](summary) x = self.decode(x, skips) logits = self.heads[device](x) edge = F.interpolate( edge, size=logits.shape[-1], mode="linear", align_corners=True ) return torch.cat( [logits[:, :, :1] + edge[:, :, None, :], logits[:, :, 1:]], dim=2 ) class SpatialBoundaryNet(AxialBoundaryNet): def __init__( self, boundary_keys, base=16, in_channels=1, layers=2, lateral_layers=2, heads=6 ): super().__init__(boundary_keys, base, in_channels, layers, heads) if not 1 <= lateral_layers <= 4: raise ValueError("Lateral context requires one to four layers") self.lateral = nn.ModuleList( [AxialContext(12 * base, heads) for _ in range(lateral_layers)] ) def forward(self, x, device): x, skips = self.encode(x) for context in self.axial: x = context(x) for context in self.lateral: x = context(x.transpose(-1, -2)).transpose(-1, -2).contiguous() x = self.decode(x, skips) return self.heads[device](x) def native_coordinate_weight(original_heights, model_height, weight, device=None): if model_height < 2 or not math.isfinite(weight) or weight < 0: raise ValueError("Invalid coordinate-loss geometry or coefficient") heights = torch.as_tensor(original_heights, device=device, dtype=torch.float32) if heights.ndim != 1 or not torch.isfinite(heights).all() or (heights < 2).any(): raise ValueError("Invalid original structural heights") return ((heights - 1) / (model_height - 1) * weight)[:, None, None] def expand_centered_channels(state, source_channels, target_channels): if ( source_channels < 1 or source_channels % 2 != 1 or target_channels % 2 != 1 or target_channels < source_channels ): raise ValueError( "Only odd, nonshrinking centered channel expansions are supported" ) key = "encoder.0.net.0.weight" weight = state[key] if weight.shape[1] != source_channels: raise ValueError("Source input-weight geometry mismatch") if source_channels == target_channels: return dict(state) expanded = weight.new_zeros((weight.shape[0], target_channels, *weight.shape[2:])) offset = (target_channels - source_channels) // 2 expanded[:, offset : offset + source_channels] = weight result = dict(state) result[key] = expanded return result