JECT_224 / ject_encoder.py
lukeingawesome's picture
Initial release
fb8046d verified
Raw History Blame Contribute Delete
34.8 kB
"""JECT image encoder: standalone architecture definition.
Depends only on ``torch``. Defines the 3D Swin-UNETR-style hierarchical encoder,
the dense latent decoder on top of it, and the config needed to rebuild both from
a training checkpoint. Training-only components (EMA, predictors, text heads,
losses) are not included.
Encoder layout (224 input, patch 4, `huge` variant)::
volume [B,1,224,224,224]
-> PatchEmbed3D (conv4x4x4) -> e0 56^3 x 128
-> stage1 (2x swin, window 7) -> e1 56^3 x 128
-> PatchMerge -> stage2 (4x swin) -> e2 28^3 x 256
-> PatchMerge -> +pos_embed -> stage3 (9x swin) -> e3 14^3 x 512
-> PatchMerge -> +pos_embed_s4 -> stage4 (9x global attn) -> e4 7^3 x 1024
At 336 input the grids are 84/42/21/11. The learned s3/s4 absolute position
embeddings are resolution-specific, so each input size has its own checkpoint.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Dict, Optional, Tuple
import torch
import torch.nn as nn
import torch.nn.functional as F
Triple = Tuple[int, int, int]
# Per-variant (dims, heads, depths) for scales (s1, s2, s3, s4).
VARIANTS = {
"tiny": dict(dims=(32, 64, 128, 256), heads=(2, 4, 8, 8), depths=(2, 2, 2, 2)),
"small": dict(dims=(48, 96, 192, 384), heads=(3, 6, 12, 12), depths=(2, 2, 4, 2)),
"base": dict(dims=(64, 128, 256, 512), heads=(4, 8, 16, 16), depths=(2, 2, 4, 2)),
"large": dict(dims=(96, 192, 384, 768), heads=(6, 12, 24, 24), depths=(2, 2, 8, 4)),
"huge": dict(dims=(128, 256, 512, 1024), heads=(8, 16, 32, 32), depths=(2, 4, 9, 9)),
}
# HU windows the encoder was trained with. `window_id` is an additive embedding on the
# patch tokens, so the id passed at inference must match the window actually applied.
WINDOW_PARAMS = {
0: {"center": 450, "width": 2100}, # lung: -600 .. 1500
1: {"center": 195, "width": 310}, # mediastinal: 40 .. 350
2: {"center": 0, "width": 2000}, # general: -1000 .. 1000 (default)
3: {"center": 1500, "width": 5000}, # full: -1000 .. 4000
}
GENERAL_WINDOW_ID = 2
# ct_type to additive embedding index.
CT_TYPE_MAP = {"non-contrast": 0, "contrast": 1, "ldct": 2}
DEFAULT_CT_TYPE = 0
def _triple(x) -> Triple:
if isinstance(x, int):
return (x, x, x)
vals = tuple(int(v) for v in x)
if len(vals) == 1:
return (vals[0],) * 3
if len(vals) != 3:
raise ValueError(f"expected an int or a 1/3-length iterable, got {x!r}")
return vals
# ---------------------------------------------------------------------------
# Config
# ---------------------------------------------------------------------------
@dataclass
class EncoderConfig:
"""Encoder geometry and width; normally built with ``from_training_config``."""
input_size: Triple = (224, 224, 224)
patch_size: Triple = (4, 4, 4)
in_channels: int = 1
variant: str = "huge"
local_block_type: str = "swin" # stages 1-2
stage3_block_type: str = "swin"
stage4_block_type: str = "global" # full attention over the 7^3 / 11^3 grid
window: Triple = (7, 7, 7)
mlp_ratio: float = 4.0
drop: float = 0.0
num_ct_types: int = 3
# Cumulative bottom-up skips of pooled finer features into the s2/s3/s4 stage entries.
skip_fine: bool = False
# Pooling for the s3 skip: "maxmean" ([max, mean] cell pools) or "attention" (LocalAttentionPool3D).
skip_pool_s3: str = "maxmean"
# Pooling for the s2 and s4 skips: "maxmean" or "attention".
skip_pool_s2s4: str = "maxmean"
# filled by __post_init__
dims: Tuple[int, ...] = field(default_factory=tuple)
heads: Tuple[int, ...] = field(default_factory=tuple)
depths: Tuple[int, ...] = field(default_factory=tuple)
grid_s1: Triple = (0, 0, 0)
grid_s2: Triple = (0, 0, 0)
grid_s3: Triple = (0, 0, 0)
grid_s4: Triple = (0, 0, 0)
def __post_init__(self):
self.input_size = _triple(self.input_size)
self.patch_size = _triple(self.patch_size)
self.window = _triple(self.window)
if self.variant not in VARIANTS:
raise ValueError(f"unknown variant {self.variant!r}; expected one of {sorted(VARIANTS)}")
spec = VARIANTS[self.variant]
if self.skip_pool_s3 not in ("maxmean", "attention"):
raise ValueError(f"unknown skip_pool_s3 {self.skip_pool_s3!r}; expected 'maxmean' or 'attention'")
if self.skip_pool_s3 != "maxmean" and not self.skip_fine:
raise ValueError("skip_pool_s3='attention' requires skip_fine (mirrors pretrain)")
if self.skip_pool_s2s4 not in ("maxmean", "attention"):
raise ValueError(f"unknown skip_pool_s2s4 {self.skip_pool_s2s4!r}; expected 'maxmean' or 'attention'")
if self.skip_pool_s2s4 != "maxmean" and not self.skip_fine:
raise ValueError("skip_pool_s2s4='attention' requires skip_fine (mirrors pretrain)")
self.dims = tuple(spec["dims"])
self.heads = tuple(spec["heads"])
self.depths = tuple(spec["depths"])
for name, v, p in zip("DHW", self.input_size, self.patch_size):
if v % p:
raise ValueError(f"input_size {name}={v} not divisible by patch_size {p}")
s1 = tuple(v // p for v, p in zip(self.input_size, self.patch_size))
s2 = tuple(g // 2 for g in s1)
s3 = tuple(g // 4 for g in s1)
# PatchMerge3D pads odd grids before halving, so s4 = ceil(s3 / 2) (e.g. s3 21 gives s4 11).
s4 = tuple((g + 1) // 2 for g in s3)
self.grid_s1, self.grid_s2, self.grid_s3, self.grid_s4 = s1, s2, s3, s4
# -- convenience ---------------------------------------------------------
@property
def num_tokens(self) -> Dict[str, int]:
return {f"s{i}": g[0] * g[1] * g[2]
for i, g in enumerate((self.grid_s1, self.grid_s2, self.grid_s3, self.grid_s4), 1)}
@property
def grids(self) -> Dict[str, Triple]:
return {"s1": self.grid_s1, "s2": self.grid_s2, "s3": self.grid_s3, "s4": self.grid_s4}
@property
def channels(self) -> Dict[str, int]:
return {f"s{i}": d for i, d in enumerate(self.dims, 1)}
@classmethod
def from_training_config(cls, saved: dict) -> "EncoderConfig":
"""Rebuild from the `config` dict stored inside a JECT training checkpoint.
The input size is derived from ``grid_size_s1 * patch_size`` rather than the ``volume``
block, and the derived grids and widths are checked against the saved ones.
"""
patch = saved.get("patch", {})
model = saved.get("model", {})
psize = _triple(patch.get("patch_size", (4, 4, 4)))
g1 = _triple(patch["grid_size_s1"])
cfg = cls(
input_size=tuple(g * p for g, p in zip(g1, psize)),
patch_size=psize,
in_channels=int(saved.get("volume", {}).get("channels", 1)),
variant=str(model.get("variant", "huge")),
local_block_type=str(model.get("local_block_type", "swin")),
stage3_block_type=str(model.get("stage3_block_type", "swin")),
stage4_block_type=str(model.get("stage4_block_type", "global")),
window=_triple(model.get("swin_window_size", (7, 7, 7))),
mlp_ratio=float(model.get("mlp_ratio", 4.0)),
drop=0.0, # no dropout at inference
num_ct_types=int(model.get("num_ct_types", 3)),
skip_fine=bool(model.get("skip_fine", False)),
skip_pool_s3=str(model.get("skip_pool_s3", "maxmean")),
skip_pool_s2s4=str(model.get("skip_pool_s2s4", "maxmean")),
)
for name, derived in (("grid_size_s2", cfg.grid_s2), ("grid_size_s3", cfg.grid_s3),
("grid_size_s4", cfg.grid_s4)):
if name in patch and _triple(patch[name]) != derived:
raise ValueError(f"geometry mismatch: checkpoint {name}={tuple(patch[name])} "
f"but input_size {cfg.input_size} implies {derived}")
for key, want in (("dim_s1", cfg.dims[0]), ("dim_s2", cfg.dims[1]),
("dim_s3", cfg.dims[2]), ("dim_s4", cfg.dims[3])):
if key in model and int(model[key]) != want:
raise ValueError(f"width mismatch: checkpoint {key}={model[key]} but "
f"variant {cfg.variant!r} implies {want}")
return cfg
# ---------------------------------------------------------------------------
# Shared pieces
# ---------------------------------------------------------------------------
class Mlp(nn.Module):
def __init__(self, dim: int, mlp_ratio: float = 4.0, drop: float = 0.0):
super().__init__()
hidden = int(dim * mlp_ratio)
self.fc1 = nn.Linear(dim, hidden)
self.act = nn.GELU()
self.fc2 = nn.Linear(hidden, dim)
self.drop = nn.Dropout(drop)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.drop(self.fc2(self.drop(self.act(self.fc1(x)))))
def to_grid(x: torch.Tensor, grid: Triple) -> torch.Tensor:
"""[B, N, C] -> [B, C, D, H, W]."""
B, N, C = x.shape
D, H, W = grid
if N != D * H * W:
raise ValueError(f"token count {N} does not match grid {grid} ({D * H * W})")
return x.transpose(1, 2).reshape(B, C, D, H, W)
def to_tokens(x: torch.Tensor) -> torch.Tensor:
"""[B, C, D, H, W] -> [B, N, C]."""
return x.flatten(2).transpose(1, 2)
class Attention(nn.Module):
"""Multi-head self-attention over all tokens (no positional bias)."""
def __init__(self, dim: int, num_heads: int, attn_drop: float = 0.0, proj_drop: float = 0.0):
super().__init__()
if dim % num_heads:
raise ValueError(f"dim={dim} not divisible by num_heads={num_heads}")
self.num_heads = num_heads
self.head_dim = dim // num_heads
self.qkv = nn.Linear(dim, dim * 3)
self.attn_drop_p = float(attn_drop)
self.proj = nn.Linear(dim, dim)
self.proj_drop = nn.Dropout(proj_drop)
def forward(self, x: torch.Tensor, bias: Optional[torch.Tensor] = None) -> torch.Tensor:
B, N, C = x.shape
qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4)
q, k, v = qkv.unbind(0)
attn_mask = bias.unsqueeze(0).to(q.dtype) if bias is not None else None
out = F.scaled_dot_product_attention(
q, k, v, attn_mask=attn_mask, dropout_p=self.attn_drop_p if self.training else 0.0)
out = out.transpose(1, 2).reshape(B, N, C)
return self.proj_drop(self.proj(out))
class TransformerBlock3D(nn.Module):
"""Pre-norm global transformer block (stage 4)."""
def __init__(self, dim: int, num_heads: int, mlp_ratio: float = 4.0, drop: float = 0.0):
super().__init__()
self.norm1 = nn.LayerNorm(dim)
self.attn = Attention(dim, num_heads, attn_drop=drop, proj_drop=drop)
self.norm2 = nn.LayerNorm(dim)
self.mlp = Mlp(dim, mlp_ratio, drop)
def forward(self, x: torch.Tensor, grid: Triple) -> torch.Tensor:
x = x + self.attn(self.norm1(x))
x = x + self.mlp(self.norm2(x))
return x
def _pad_to_window(x: torch.Tensor, window: Triple):
"""Pad a [B, D, H, W, C] grid up to a multiple of the window on each spatial axis."""
_, D, H, W, _ = x.shape
wd, wh, ww = window
pd, ph, pw = (wd - D % wd) % wd, (wh - H % wh) % wh, (ww - W % ww) % ww
if pd or ph or pw:
x = F.pad(x, (0, 0, 0, pw, 0, ph, 0, pd)) # pads W, H, D (C last, untouched)
return x, (D, H, W)
def _win_partition(x: torch.Tensor, window: Triple) -> torch.Tensor:
B, D, H, W, C = x.shape
wd, wh, ww = window
x = x.view(B, D // wd, wd, H // wh, wh, W // ww, ww, C)
x = x.permute(0, 1, 3, 5, 2, 4, 6, 7).contiguous()
return x.view(-1, wd * wh * ww, C)
def _win_merge(windows: torch.Tensor, gridp: Triple, window: Triple, B: int) -> torch.Tensor:
Dp, Hp, Wp = gridp
wd, wh, ww = window
C = windows.shape[-1]
x = windows.view(B, Dp // wd, Hp // wh, Wp // ww, wd, wh, ww, C)
x = x.permute(0, 1, 4, 2, 5, 3, 6, 7).contiguous()
return x.view(B, Dp, Hp, Wp, C)
class WindowAttention3D(nn.Module):
"""Windowed multi-head attention with a 3D relative-position-bias table."""
def __init__(self, dim: int, num_heads: int, window: Triple,
attn_drop: float = 0.0, proj_drop: float = 0.0):
super().__init__()
if dim % num_heads:
raise ValueError(f"dim={dim} not divisible by num_heads={num_heads}")
self.num_heads = num_heads
self.head_dim = dim // num_heads
self.window = window
wd, wh, ww = window
self.relative_position_bias_table = nn.Parameter(
torch.zeros((2 * wd - 1) * (2 * wh - 1) * (2 * ww - 1), num_heads))
coords = torch.stack(torch.meshgrid(
torch.arange(wd), torch.arange(wh), torch.arange(ww), indexing="ij"))
coords_flat = coords.reshape(3, -1)
rel = (coords_flat[:, :, None] - coords_flat[:, None, :]).permute(1, 2, 0).contiguous()
rel[..., 0] += wd - 1
rel[..., 1] += wh - 1
rel[..., 2] += ww - 1
rel[..., 0] *= (2 * wh - 1) * (2 * ww - 1)
rel[..., 1] *= (2 * ww - 1)
self.register_buffer("relative_position_index", rel.sum(-1).view(-1), persistent=False)
self.qkv = nn.Linear(dim, dim * 3)
self.attn_drop_p = float(attn_drop)
self.proj = nn.Linear(dim, dim)
self.proj_drop = nn.Dropout(proj_drop)
def _bias(self) -> torch.Tensor:
ws = int(round(self.relative_position_index.numel() ** 0.5))
table = self.relative_position_bias_table[self.relative_position_index]
return table.view(ws, ws, -1).permute(2, 0, 1).contiguous() # [heads, ws, ws]
def forward(self, windows: torch.Tensor) -> torch.Tensor:
Bn, ws, C = windows.shape
qkv = self.qkv(windows).reshape(Bn, ws, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4)
q, k, v = qkv.unbind(0)
attn_mask = self._bias().unsqueeze(0).to(q.dtype)
out = F.scaled_dot_product_attention(
q, k, v, attn_mask=attn_mask, dropout_p=self.attn_drop_p if self.training else 0.0)
out = out.transpose(1, 2).reshape(Bn, ws, C)
return self.proj_drop(self.proj(out))
class SwinBlock3D(nn.Module):
"""Shifted-window 3D attention block; pads internally so any grid size works."""
def __init__(self, dim: int, num_heads: int, window: Triple,
mlp_ratio: float = 4.0, drop: float = 0.0, shift: bool = False):
super().__init__()
self.window = window
self.shift_size = tuple(w // 2 for w in window) if shift else (0, 0, 0)
self.norm1 = nn.LayerNorm(dim)
self.attn = WindowAttention3D(dim, num_heads, window, attn_drop=drop, proj_drop=drop)
self.norm2 = nn.LayerNorm(dim)
self.mlp = Mlp(dim, mlp_ratio, drop)
def forward(self, x: torch.Tensor, grid: Triple) -> torch.Tensor:
B, N, C = x.shape
D, H, W = grid
h = self.norm1(x).view(B, D, H, W, C)
h, _ = _pad_to_window(h, self.window)
Dp, Hp, Wp = h.shape[1:4]
if any(s > 0 for s in self.shift_size):
h = torch.roll(h, shifts=tuple(-s for s in self.shift_size), dims=(1, 2, 3))
windows = self.attn(_win_partition(h, self.window))
h = _win_merge(windows, (Dp, Hp, Wp), self.window, B)
if any(s > 0 for s in self.shift_size):
h = torch.roll(h, shifts=self.shift_size, dims=(1, 2, 3))
h = h[:, :D, :H, :W].reshape(B, N, C) # crop the window padding
x = x + h
x = x + self.mlp(self.norm2(x))
return x
class ConvNeXtBlock3D(nn.Module):
"""Depthwise 3D ConvNeXt block."""
def __init__(self, dim: int, mlp_ratio: float = 4.0, drop: float = 0.0):
super().__init__()
self.dwconv = nn.Conv3d(dim, dim, kernel_size=7, padding=3, groups=dim)
self.norm = nn.LayerNorm(dim)
hidden = int(dim * mlp_ratio)
self.pw1 = nn.Linear(dim, hidden)
self.act = nn.GELU()
self.pw2 = nn.Linear(hidden, dim)
self.gamma = nn.Parameter(1e-6 * torch.ones(dim))
def forward(self, x: torch.Tensor, grid: Triple) -> torch.Tensor:
y = to_tokens(self.dwconv(to_grid(x, grid)))
y = self.pw2(self.act(self.pw1(self.norm(y))))
return x + self.gamma * y
def build_block(block_type: str, dim: int, num_heads: int, window: Triple,
mlp_ratio: float, drop: float, shift: bool = False) -> nn.Module:
if block_type == "swin":
return SwinBlock3D(dim, num_heads, window, mlp_ratio, drop, shift=shift)
if block_type in ("transformer", "global"):
return TransformerBlock3D(dim, num_heads, mlp_ratio, drop)
if block_type == "convnext":
return ConvNeXtBlock3D(dim, mlp_ratio, drop)
raise ValueError(f"unknown block type {block_type!r}")
class PatchEmbed3D(nn.Module):
def __init__(self, channels: int, dim: int, patch_size: Triple):
super().__init__()
self.proj = nn.Conv3d(channels, dim, kernel_size=patch_size, stride=patch_size)
self.norm = nn.LayerNorm(dim)
def forward(self, volume: torch.Tensor) -> torch.Tensor:
return self.norm(to_tokens(self.proj(volume)))
def _pool_fine(x: torch.Tensor, grid: Triple, grid_to: Triple) -> torch.Tensor:
"""Max- and mean-pool z-major tokens on `grid` into the cells of `grid_to`.
[B, N, C] -> [B, N_to, 2C], pooling the k^3 fine tokens of each coarse cell (k = grid / grid_to)."""
B, N, C = x.shape
D, H, W = grid
k = D // grid_to[0]
if (D, H, W) != (k * grid_to[0], k * grid_to[1], k * grid_to[2]):
raise ValueError(f"skip_fine needs grid {grid} to be an integer multiple of the coarser grid {grid_to}")
v = x.transpose(1, 2).reshape(B, C, D, H, W)
return torch.cat([F.max_pool3d(v, k), F.avg_pool3d(v, k)], dim=1).flatten(2).transpose(1, 2)
class LocalAttentionPool3D(nn.Module):
"""Coarse-query attention over one cell's fine tokens, with a learned 3D-offset bias.
Four heads, 128 internal channels, and the same 2C output width as max/mean pooling.
Token order within each cell is z/y/x, so each bias index is a distinct relative 3D
position. Attention never crosses cells.
"""
SDPA_CHUNK = 32768 # max cells per SDPA call
def __init__(self, fine_dim: int, coarse_dim: int, factor: int):
super().__init__()
self.factor = factor
self.heads, self.head_dim = 4, 32
self.fine_norm = nn.LayerNorm(fine_dim)
self.query_norm = nn.LayerNorm(coarse_dim)
self.query = nn.Linear(coarse_dim, 128)
self.key_value = nn.Linear(fine_dim, 256)
self.relative_bias = nn.Parameter(torch.zeros(4, factor ** 3))
self.output = nn.Linear(128, 2 * fine_dim)
def forward(self, fine: torch.Tensor, coarse: torch.Tensor, grid: Triple,
grid_to: Triple) -> torch.Tensor:
B, N, C = fine.shape
D, H, W = grid_to
k = self.factor
if tuple(grid) != (D * k, H * k, W * k) or N != D * H * W * k ** 3:
raise ValueError("attention skip requires exactly nested fine/coarse grids")
if coarse.shape[:2] != (B, D * H * W):
raise ValueError("attention skip query count does not match its coarse grid")
cells = fine.reshape(B, D, k, H, k, W, k, C)
cells = cells.permute(0, 1, 3, 5, 2, 4, 6, 7).reshape(B * D * H * W, k ** 3, C)
kv = self.key_value(self.fine_norm(cells))
kv = kv.reshape(-1, k ** 3, 2, self.heads, self.head_dim).permute(2, 0, 3, 1, 4)
q = self.query(self.query_norm(coarse)).reshape(-1, self.heads, 1, self.head_dim)
bias = self.relative_bias.to(q.dtype)[None, :, None, :]
# One SDPA batch entry per cell (B*D*H*W). Above ~65k entries the fused kernel exceeds the CUDA
# launch-grid limit, so large inputs are processed in chunks; the result is unchanged.
n = q.shape[0]
if n <= self.SDPA_CHUNK:
pooled = F.scaled_dot_product_attention(q, kv[0], kv[1], attn_mask=bias, dropout_p=0.0)
else:
pooled = torch.cat([F.scaled_dot_product_attention(q[i:i + self.SDPA_CHUNK], kv[0][i:i + self.SDPA_CHUNK],
kv[1][i:i + self.SDPA_CHUNK], attn_mask=bias, dropout_p=0.0)
for i in range(0, n, self.SDPA_CHUNK)], dim=0)
return self.output(pooled.reshape(B, D * H * W, 128))
class PatchMerge3D(nn.Module):
"""2x2x2 patch merging: [B, N, C] -> [B, N/8, out_dim] (odd grids are padded first)."""
def __init__(self, in_dim: int, out_dim: int):
super().__init__()
self.norm = nn.LayerNorm(8 * in_dim)
self.reduce = nn.Linear(8 * in_dim, out_dim, bias=False)
def forward(self, x: torch.Tensor, grid: Triple) -> Tuple[torch.Tensor, Triple]:
B, N, C = x.shape
D, H, W = grid
if D % 2 or H % 2 or W % 2:
x = x.view(B, D, H, W, C)
x = F.pad(x, (0, 0, 0, W % 2, 0, H % 2, 0, D % 2))
D, H, W = D + D % 2, H + H % 2, W + W % 2
x = x.reshape(B, D * H * W, C)
x = x.view(B, D // 2, 2, H // 2, 2, W // 2, 2, C)
x = x.permute(0, 1, 3, 5, 2, 4, 6, 7).contiguous()
x = x.view(B, (D // 2) * (H // 2) * (W // 2), 8 * C)
return self.reduce(self.norm(x)), (D // 2, H // 2, W // 2)
# ---------------------------------------------------------------------------
# Encoder / dense decoder
# ---------------------------------------------------------------------------
class JECTEncoder(nn.Module):
"""Hierarchical 3D encoder producing token features at the s1/s2/s3/s4 grids.
Module and parameter names match the pretraining checkpoint exactly, so
``state_dict`` keys load without remapping (strip the ``target_encoder.``
or ``student_encoder.`` prefix first).
"""
def __init__(self, config: EncoderConfig):
super().__init__()
self.config = config
d1, d2, d3, d4 = config.dims
h1, h2, h3, h4 = config.heads
n1, n2, n3, n4 = config.depths
win = config.window
self.grid_s1, self.grid_s2 = config.grid_s1, config.grid_s2
self.grid_s3, self.grid_s4 = config.grid_s3, config.grid_s4
self.patch_embed = PatchEmbed3D(config.in_channels, d1, config.patch_size)
self.window_embed = nn.Embedding(4, d1) # HU-window id (see WINDOW_PARAMS)
self.ct_type_embed = nn.Embedding(config.num_ct_types, d1)
self.mask_token = nn.Parameter(torch.zeros(1, 1, d1)) # pretraining only; unused here
self.stage1 = nn.ModuleList([
build_block(config.local_block_type, d1, h1, win, config.mlp_ratio, config.drop,
shift=(i % 2 == 1)) for i in range(n1)])
self.merge1 = PatchMerge3D(d1, d2)
self.stage2 = nn.ModuleList([
build_block(config.local_block_type, d2, h2, win, config.mlp_ratio, config.drop,
shift=(i % 2 == 1)) for i in range(n2)])
self.merge2 = PatchMerge3D(d2, d3)
# skip_fine: at each stage entry, a projection of pooled features from every finer representation
# (patch embeddings, e1, e2, e3).
self.skip_fine = bool(config.skip_fine)
if self.skip_fine:
self.skip2_norm = nn.LayerNorm(2 * (d1 + d1)); self.skip2_proj = nn.Linear(2 * (d1 + d1), d2)
self.skip3_norm = nn.LayerNorm(2 * (d1 + d1 + d2)); self.skip3_proj = nn.Linear(2 * (d1 + d1 + d2), d3)
self.skip4_norm = nn.LayerNorm(2 * (d1 + d1 + d2 + d3)); self.skip4_proj = nn.Linear(2 * (d1 + d1 + d2 + d3), d4)
# skip_pool_s3="attention": one LocalAttentionPool3D per s3 skip source instead of max/mean pooling.
self.skip_pool_s3 = str(config.skip_pool_s3)
if self.skip_fine and self.skip_pool_s3 == "attention":
self.skip3_attention = nn.ModuleList([
LocalAttentionPool3D(d1, d3, 4),
LocalAttentionPool3D(d1, d3, 4),
LocalAttentionPool3D(d2, d3, 2),
])
# skip_pool_s2s4="attention": the same per-cell attention pooling at the s2 and s4 entries.
self.skip_pool_s2s4 = str(getattr(config, "skip_pool_s2s4", "maxmean"))
if self.skip_fine and self.skip_pool_s2s4 == "attention":
self.skip2_attention = nn.ModuleList([
LocalAttentionPool3D(d1, d2, 2),
LocalAttentionPool3D(d1, d2, 2),
])
self.skip4_attention = nn.ModuleList([
LocalAttentionPool3D(d1, d4, 8),
LocalAttentionPool3D(d1, d4, 8),
LocalAttentionPool3D(d2, d4, 4),
LocalAttentionPool3D(d3, d4, 2),
])
self.pos_embed = nn.Parameter(torch.zeros(1, config.num_tokens["s3"], d3))
self.stage3 = nn.ModuleList([
build_block(config.stage3_block_type, d3, h3, win, config.mlp_ratio, config.drop,
shift=(i % 2 == 1)) for i in range(n3)])
self.merge3 = PatchMerge3D(d3, d4)
self.pos_embed_s4 = nn.Parameter(torch.zeros(1, config.num_tokens["s4"], d4))
self.stage4 = nn.ModuleList([
build_block(config.stage4_block_type, d4, h4, win, config.mlp_ratio, config.drop,
shift=(i % 2 == 1)) for i in range(n4)])
def trunk(self, volume: torch.Tensor,
window_id: Optional[torch.Tensor] = None,
ct_type_id: Optional[torch.Tensor] = None,
mask_s1: Optional[torch.Tensor] = None) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Run stages 1-3 and merge3 (with skips), stopping before pos_embed_s4 and stage 4.
Useful for fine-tuning only stage 4. Returns ``(x_s4_in, e1, e2, e3)``.
"""
expected = self.config.input_size
if tuple(volume.shape[-3:]) != tuple(expected):
raise ValueError(f"expected volume {expected}, got {tuple(volume.shape[-3:])}. "
f"This checkpoint is fixed-resolution (learned s3/s4 position "
f"embeddings); use the checkpoint matching your input size.")
x = self.patch_embed(volume)
if window_id is not None:
x = x + self.window_embed(window_id).unsqueeze(1)
if ct_type_id is not None:
x = x + self.ct_type_embed(ct_type_id).unsqueeze(1)
if mask_s1 is not None:
x = torch.where(mask_s1.unsqueeze(-1), self.mask_token.to(x.dtype), x)
p0 = x if self.skip_fine else None
for blk in self.stage1:
x = blk(x, self.grid_s1)
e1 = x
x, _ = self.merge1(x, self.grid_s1)
if self.skip_fine:
if self.skip_pool_s2s4 == "attention":
sources = ((p0, self.grid_s1), (e1, self.grid_s1))
fine = torch.cat([pool(f, x, grid, self.grid_s2)
for pool, (f, grid) in zip(self.skip2_attention, sources)], dim=-1)
else:
fine = torch.cat([_pool_fine(p0, self.grid_s1, self.grid_s2), _pool_fine(e1, self.grid_s1, self.grid_s2)], dim=-1)
x = x + self.skip2_proj(self.skip2_norm(fine)).to(x.dtype)
for blk in self.stage2:
x = blk(x, self.grid_s2)
e2 = x
x, _ = self.merge2(x, self.grid_s2)
if self.skip_fine:
if self.skip_pool_s3 == "attention":
sources = ((p0, self.grid_s1), (e1, self.grid_s1), (e2, self.grid_s2))
fine = torch.cat([pool(f, x, grid, self.grid_s3)
for pool, (f, grid) in zip(self.skip3_attention, sources)], dim=-1)
else:
fine = torch.cat([_pool_fine(p0, self.grid_s1, self.grid_s3), _pool_fine(e1, self.grid_s1, self.grid_s3),
_pool_fine(e2, self.grid_s2, self.grid_s3)], dim=-1)
x = x + self.skip3_proj(self.skip3_norm(fine)).to(x.dtype)
x = x + self.pos_embed
for blk in self.stage3:
x = blk(x, self.grid_s3)
e3 = x
x, _ = self.merge3(x, self.grid_s3)
if self.skip_fine:
if self.skip_pool_s2s4 == "attention":
sources = ((p0, self.grid_s1), (e1, self.grid_s1), (e2, self.grid_s2), (e3, self.grid_s3))
fine = torch.cat([pool(f, x, grid, self.grid_s4)
for pool, (f, grid) in zip(self.skip4_attention, sources)], dim=-1)
else:
fine = torch.cat([_pool_fine(p0, self.grid_s1, self.grid_s4), _pool_fine(e1, self.grid_s1, self.grid_s4),
_pool_fine(e2, self.grid_s2, self.grid_s4), _pool_fine(e3, self.grid_s3, self.grid_s4)], dim=-1)
x = x + self.skip4_proj(self.skip4_norm(fine)).to(x.dtype)
return x, e1, e2, e3
def forward(self, volume: torch.Tensor,
window_id: Optional[torch.Tensor] = None,
ct_type_id: Optional[torch.Tensor] = None,
mask_s1: Optional[torch.Tensor] = None) -> Dict[str, torch.Tensor]:
"""Encode a volume.
Args:
volume: [B, 1, D, H, W], HU-windowed to [0, 1] (see ``preprocess``).
window_id: [B] long, the HU window applied (see WINDOW_PARAMS).
ct_type_id: [B] long, optional (see CT_TYPE_MAP).
mask_s1: [B, N_s1] bool, optional; replaces tokens with the mask token.
Returns:
dict of token features ``{e1, e2, e3, e4}``, each [B, N_s, C_s].
"""
x, e1, e2, e3 = self.trunk(volume, window_id, ct_type_id, mask_s1)
x = x + self.pos_embed_s4
for blk in self.stage4:
x = blk(x, self.grid_s4)
e4 = x
return {"e1": e1, "e2": e2, "e3": e3, "e4": e4}
class DenseLatentDecoder(nn.Module):
"""Upsampling decoder producing dense latents at the s3/s2/s1 grids.
These latents are the JEPA targets during pretraining. Optional at inference; the
encoder features are the recommended readout for probing and retrieval.
"""
def __init__(self, config: EncoderConfig):
super().__init__()
d1, d2, d3, _ = config.dims
self.grid_s1, self.grid_s2, self.grid_s3 = config.grid_s1, config.grid_s2, config.grid_s3
self.block3 = Mlp(d3, config.mlp_ratio, config.drop)
self.norm3 = nn.LayerNorm(d3)
self.up3to2 = nn.Linear(d3, d2)
self.fuse2 = nn.LayerNorm(d2)
self.block2 = Mlp(d2, config.mlp_ratio, config.drop)
self.up2to1 = nn.Linear(d2, d1)
self.fuse1 = nn.LayerNorm(d1)
self.block1 = Mlp(d1, config.mlp_ratio, config.drop)
self.out1 = nn.LayerNorm(d1)
self.out2 = nn.LayerNorm(d2)
self.out3 = nn.LayerNorm(d3)
@staticmethod
def _upsample(x: torch.Tensor, src_grid: Triple, dst_grid: Triple) -> torch.Tensor:
y = F.interpolate(to_grid(x, src_grid), size=tuple(dst_grid),
mode="trilinear", align_corners=False)
return to_tokens(y)
def forward(self, enc: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
e1, e2, e3 = enc["e1"], enc["e2"], enc["e3"]
d3 = e3 + self.block3(self.norm3(e3))
u2 = self.up3to2(self._upsample(d3, self.grid_s3, self.grid_s2))
d2 = u2 + e2
d2 = d2 + self.block2(self.fuse2(d2))
u1 = self.up2to1(self._upsample(d2, self.grid_s2, self.grid_s1))
d1 = u1 + e1
d1 = d1 + self.block1(self.fuse1(d1))
return {"s1": self.out1(d1), "s2": self.out2(d2), "s3": self.out3(d3)}
# ---------------------------------------------------------------------------
# Checkpoint loading
# ---------------------------------------------------------------------------
def _substate(state: Dict[str, torch.Tensor], prefix: str) -> Dict[str, torch.Tensor]:
p = prefix if prefix.endswith(".") else prefix + "."
return {k[len(p):]: v for k, v in state.items() if k.startswith(p)}
def load_pretrained(checkpoint_path, stream: str = "target", with_decoder: bool = False,
map_location: str = "cpu"):
"""Rebuild the encoder (and optionally the dense decoder) from a training checkpoint.
Args:
checkpoint_path: path to ``checkpoint_epoch_*.pt``.
stream: ``"target"`` (EMA teacher, recommended) or ``"student"`` (online weights).
with_decoder: also build and load the dense latent decoder.
map_location: device passed to ``torch.load``.
Returns:
``(encoder, config, info)``, or ``(encoder, decoder, config, info)`` when
``with_decoder``. Modules are in eval mode. ``info`` holds the epoch, step,
stream, checkpoint path and the raw training config.
"""
if stream not in ("target", "student"):
raise ValueError(f"stream must be 'target' or 'student', got {stream!r}")
ck = torch.load(str(checkpoint_path), map_location=map_location, weights_only=False)
if "model" not in ck or "config" not in ck:
raise ValueError(f"{checkpoint_path} is not an JECT training checkpoint "
f"(missing 'model'/'config'; found {sorted(ck)[:6]})")
cfg = EncoderConfig.from_training_config(ck["config"])
encoder = JECTEncoder(cfg)
enc_state = _substate(ck["model"], f"{stream}_encoder")
if not enc_state:
raise ValueError(f"checkpoint has no '{stream}_encoder.*' weights")
missing, unexpected = encoder.load_state_dict(enc_state, strict=False)
# `relative_position_index` is a non-persistent buffer rebuilt in __init__.
missing = [k for k in missing if not k.endswith("relative_position_index")]
if missing or unexpected:
raise RuntimeError(f"encoder state mismatch: missing={missing[:5]} "
f"unexpected={unexpected[:5]}")
encoder.eval()
info = {"epoch": ck.get("epoch"), "step": ck.get("step"), "stream": stream,
"checkpoint": str(checkpoint_path), "training_config": ck["config"]}
if not with_decoder:
return encoder, cfg, info
decoder = DenseLatentDecoder(cfg)
dec_state = _substate(ck["model"], f"{stream}_decoder")
if not dec_state:
raise ValueError(f"checkpoint has no '{stream}_decoder.*' weights")
missing, unexpected = decoder.load_state_dict(dec_state, strict=True)
decoder.eval()
return encoder, decoder, cfg, info