farrell236's picture
Deploy OCT viewer runtime and bundled checkpoint
bf88397 verified
Raw History Blame Contribute Delete
12.5 kB
"""Boundary U-Nets, image context, axial decisions, and training losses."""
from __future__ import annotations
import math
import torch
from torch import nn
from torch.nn import functional as F
class Block(nn.Module):
def __init__(self, cin, cout):
super().__init__()
groups = min(8, cout)
self.net = nn.Sequential(
nn.Conv2d(cin, cout, 3, padding=1, bias=False),
nn.GroupNorm(groups, cout),
nn.SiLU(inplace=True),
nn.Conv2d(cout, cout, 3, padding=1, bias=False),
nn.GroupNorm(groups, cout),
nn.SiLU(inplace=True),
)
def forward(self, x):
return self.net(x)
class BoundaryNet(nn.Module):
def __init__(self, boundary_keys, base=16, in_channels=1):
super().__init__()
self.boundary_keys = boundary_keys
channels = [base, 2 * base, 4 * base, 8 * base, 12 * base]
self.encoder = nn.ModuleList(
[
Block(in_channels if i == 0 else channels[i - 1], count)
for i, count in enumerate(channels)
]
)
self.decoder = nn.ModuleList(
[
Block(channels[i] + channels[i - 1], channels[i - 1])
for i in range(4, 0, -1)
]
)
self.heads = nn.ModuleDict(
{d: nn.Conv2d(base, len(keys), 1) for d, keys in boundary_keys.items()}
)
def encode(self, x):
"""Return the bottleneck and skip features without changing state keys."""
skips = []
for i, enc in enumerate(self.encoder):
if i:
x = F.avg_pool2d(x, 2)
x = enc(x)
skips.append(x)
return x, skips
def decode(self, x, skips):
"""Reconstruct image features from a bottleneck and encoder skips."""
for dec, skip in zip(self.decoder, reversed(skips[:-1])):
x = F.interpolate(
x, size=skip.shape[-2:], mode="bilinear", align_corners=False
)
x = dec(torch.cat([x, skip], dim=1))
return x
def forward(self, x, device):
x, skips = self.encode(x)
x = self.decode(x, skips)
return self.heads[device](x)
def probability_median(probability):
"""Median of normalized, piecewise-uniform row bins; preserve endpoints."""
p = probability.float()
height = p.shape[-2]
cdf = p.cumsum(-2)
index = (cdf >= 0.5).to(torch.int64).argmax(-2, keepdim=True)
previous = cdf.gather(-2, (index - 1).clamp_min(0))
previous = torch.where(index == 0, torch.zeros_like(previous), previous)
mass = p.gather(-2, index).clamp_min(1e-12)
fraction = ((0.5 - previous) / mass).clamp(0, 1)
return (index.float() - 0.5 + fraction).clamp(0, height - 1).squeeze(-2)
def decode_logits(logits, local_radius=6, method="local_centroid"):
"""Decode axial likelihoods into resized row coordinates."""
z = logits.float()
h = z.shape[-2]
if method == "posterior_median":
return probability_median(z.softmax(dim=-2))
if method == "quadratic_peak":
if h < 3:
raise ValueError("Quadratic decoding needs at least three axial rows")
peak = z.argmax(dim=-2, keepdim=True)
center = peak.clamp(1, h - 2)
left = z.gather(-2, center - 1)
middle = z.gather(-2, center)
right = z.gather(-2, center + 1)
curvature = left - 2 * middle + right
concave = curvature < (-1e-6)
denominator = torch.where(concave, curvature, -torch.ones_like(curvature))
offset = 0.5 * (left - right) / denominator
fitted = torch.where(concave, center.float() + offset, peak.float())
fitted = torch.minimum(
torch.maximum(fitted, peak.float() - 0.5), peak.float() + 0.5
)
return fitted.clamp(0, h - 1).squeeze(-2)
if method != "local_centroid":
raise ValueError("Unknown axial likelihood decoder")
grid = torch.arange(h, device=z.device, dtype=torch.float32)[None, None, :, None]
if local_radius:
peak = z.argmax(dim=-2, keepdim=True)
z = z.masked_fill((grid - peak).abs() > local_radius, -1e4)
return (z.softmax(dim=-2) * grid).sum(dim=-2)
def median_bin_targets(target, height):
"""Two row bins whose piecewise-uniform median is the specified coordinate."""
y = target.float().clamp(0, height - 1)
lower = y.floor().to(torch.int64)
fraction = y - lower
left = torch.where(
fraction <= 0.5, 0.5 / (fraction + 0.5), (1 - fraction) / (1.5 - fraction)
)
weights = torch.zeros(
(*y.shape[:-1], height, y.shape[-1]), device=y.device, dtype=torch.float32
)
weights.scatter_add_(-2, lower.unsqueeze(-2), left.unsqueeze(-2))
weights.scatter_add_(
-2, (lower + 1).clamp_max(height - 1).unsqueeze(-2), (1 - left).unsqueeze(-2)
)
return weights
def boundary_loss(
logits,
target,
valid,
sigma=1.2,
coordinate_weight=0.01,
coordinate_loss="mean_l1",
target_encoding="gaussian",
):
"""Gaussian axial cross entropy with per-boundary support normalization."""
h = logits.shape[-2]
grid = torch.arange(h, device=logits.device, dtype=torch.float32)[
None, None, :, None
]
if target_encoding == "gaussian":
weights = torch.exp(-0.5 * ((grid - target[:, :, None, :]) / sigma) ** 2)
weights /= weights.sum(dim=2, keepdim=True).clamp_min(1e-8)
elif target_encoding == "median_bins":
weights = median_bin_targets(target, h)
else:
raise ValueError("Unknown boundary target encoding")
ce = -(weights * logits.float().log_softmax(dim=2)).sum(dim=2)
probabilities = logits.float().softmax(dim=2)
if coordinate_loss == "mean_l1":
distance = ((probabilities * grid).sum(dim=2) - target).abs()
elif coordinate_loss == "expected_abs":
distance = (probabilities * (grid - target[:, :, None, :]).abs()).sum(dim=2)
elif coordinate_loss == "median_l1":
distance = (decode_logits(logits, method="posterior_median") - target).abs()
else:
raise ValueError("Unknown coordinate loss")
loss = ce + coordinate_weight * distance
counts = valid.sum(dim=-1)
terms = (loss * valid).sum(dim=-1) / counts.clamp_min(1)
return (terms * (counts > 0)).sum() / (counts > 0).sum().clamp_min(1)
class AxialContext(nn.Module):
def __init__(self, channels, heads=6):
super().__init__()
if channels % heads:
raise ValueError("Axial attention channels must divide into heads")
self.heads = heads
self.position = nn.Conv2d(channels, channels, 3, padding=1, groups=channels)
self.norm = nn.GroupNorm(8, channels)
self.qkv = nn.Conv2d(channels, 3 * channels, 1, bias=False)
self.projection = nn.Conv2d(channels, channels, 1)
self.ff_norm = nn.GroupNorm(8, channels)
self.ff = nn.Sequential(
nn.Conv2d(channels, 2 * channels, 1),
nn.GELU(),
nn.Conv2d(2 * channels, channels, 1),
)
# Own-checkpoint expansion initially leaves all original predictions intact.
nn.init.zeros_(self.projection.weight)
nn.init.zeros_(self.projection.bias)
nn.init.zeros_(self.ff[-1].weight)
nn.init.zeros_(self.ff[-1].bias)
def forward(self, x):
n, c, h, w = x.shape
dim = c // self.heads
packed = (
self.qkv(self.norm(x + self.position(x)))
.reshape(n, 3, self.heads, dim, h, w)
.permute(1, 0, 5, 2, 4, 3)
.reshape(3, n * w, self.heads, h, dim)
)
q, k, v = packed.unbind(0)
context = F.scaled_dot_product_attention(q, k, v, dropout_p=0.0)
context = (
context.reshape(n, w, self.heads, h, dim)
.permute(0, 2, 4, 3, 1)
.contiguous()
.reshape(n, c, h, w)
)
x = x + self.projection(context)
return x + self.ff(self.ff_norm(x))
class AxialBoundaryNet(BoundaryNet):
def __init__(self, boundary_keys, base=16, in_channels=1, layers=2, heads=6):
super().__init__(boundary_keys, base, in_channels)
if not 1 <= layers <= 4:
raise ValueError("Axial context requires one to four layers")
self.axial = nn.ModuleList(
[AxialContext(12 * base, heads) for _ in range(layers)]
)
def forward(self, x, device):
x, skips = self.encode(x)
for context in self.axial:
x = context(x)
x = self.decode(x, skips)
return self.heads[device](x)
class GlobalBoundaryNet(BoundaryNet):
def __init__(self, boundary_keys, base=16, in_channels=1, layers=0, heads=6):
super().__init__(boundary_keys, base, in_channels)
if not 0 <= layers <= 4:
raise ValueError("Unsupported axial context depth")
self.axial = nn.ModuleList(
[AxialContext(12 * base, heads) for _ in range(layers)]
)
self.edge_heads = nn.ModuleDict(
{
d: nn.Sequential(
nn.Conv1d(24 * base, 8 * base, 1),
nn.GroupNorm(8, 8 * base),
nn.SiLU(),
nn.Conv1d(8 * base, 8 * base, 3, padding=1),
nn.SiLU(),
nn.Conv1d(8 * base, len(keys), 1),
)
for d, keys in boundary_keys.items()
}
)
# Preserve the original model exactly when expanding an own checkpoint.
for head in self.edge_heads.values():
nn.init.zeros_(head[-1].weight)
nn.init.zeros_(head[-1].bias)
def forward(self, x, device):
x, skips = self.encode(x)
for context in self.axial:
x = context(x)
summary = torch.cat([x.mean(dim=2), x.amax(dim=2)], dim=1)
edge = self.edge_heads[device](summary)
x = self.decode(x, skips)
logits = self.heads[device](x)
edge = F.interpolate(
edge, size=logits.shape[-1], mode="linear", align_corners=True
)
return torch.cat(
[logits[:, :, :1] + edge[:, :, None, :], logits[:, :, 1:]], dim=2
)
class SpatialBoundaryNet(AxialBoundaryNet):
def __init__(
self, boundary_keys, base=16, in_channels=1, layers=2, lateral_layers=2, heads=6
):
super().__init__(boundary_keys, base, in_channels, layers, heads)
if not 1 <= lateral_layers <= 4:
raise ValueError("Lateral context requires one to four layers")
self.lateral = nn.ModuleList(
[AxialContext(12 * base, heads) for _ in range(lateral_layers)]
)
def forward(self, x, device):
x, skips = self.encode(x)
for context in self.axial:
x = context(x)
for context in self.lateral:
x = context(x.transpose(-1, -2)).transpose(-1, -2).contiguous()
x = self.decode(x, skips)
return self.heads[device](x)
def native_coordinate_weight(original_heights, model_height, weight, device=None):
if model_height < 2 or not math.isfinite(weight) or weight < 0:
raise ValueError("Invalid coordinate-loss geometry or coefficient")
heights = torch.as_tensor(original_heights, device=device, dtype=torch.float32)
if heights.ndim != 1 or not torch.isfinite(heights).all() or (heights < 2).any():
raise ValueError("Invalid original structural heights")
return ((heights - 1) / (model_height - 1) * weight)[:, None, None]
def expand_centered_channels(state, source_channels, target_channels):
if (
source_channels < 1
or source_channels % 2 != 1
or target_channels % 2 != 1
or target_channels < source_channels
):
raise ValueError(
"Only odd, nonshrinking centered channel expansions are supported"
)
key = "encoder.0.net.0.weight"
weight = state[key]
if weight.shape[1] != source_channels:
raise ValueError("Source input-weight geometry mismatch")
if source_channels == target_channels:
return dict(state)
expanded = weight.new_zeros((weight.shape[0], target_channels, *weight.shape[2:]))
offset = (target_channels - source_channels) // 2
expanded[:, offset : offset + source_channels] = weight
result = dict(state)
result[key] = expanded
return result