"""Networks of the FM-DiT and DP (DDPM-DiT) baselines: the AdaLN DiT action backbone and the camera + state encoder. Ported from DextraVerse's imitation-learning core (`dextraverse/algorithms/il/models/nets/dit.py`, `layers.py`, `positional_embedding.py`, `vision/backbones.py`, `vision/encoder.py`), unchanged in architecture and initialization. Left out: the ROVE improvement embedding, the spatial-softmax / R3M / tactile options and the inpainting path (the baselines always condition through the global observation feature). Added: the ResNet's normalization choice (`get_resnet`): DextraVerse's GroupNorm for a backbone trained from scratch, or BatchNorm frozen at the pretrained statistics (the DETR / ACT practice) for an ImageNet-initialized backbone that is fine-tuned with the policy. """ from __future__ import annotations import math import torch import torch.nn as nn import torch.nn.functional as F import torchvision IMAGENET_MEAN = (0.485, 0.456, 0.406) IMAGENET_STD = (0.229, 0.224, 0.225) class SinusoidalPosEmb(nn.Module): def __init__(self, dim): super().__init__() self.dim = dim def forward(self, x): half_dim = self.dim // 2 emb = math.log(10000) / (half_dim - 1) emb = torch.exp(torch.arange(half_dim, device=x.device) * -emb) emb = x[:, None] * emb[None, :] return torch.cat((emb.sin(), emb.cos()), dim=-1) class RotaryPosEmb(nn.Module): """Rotary position embedding over each attention head's (S, head_dim) slice.""" def __init__(self, dim): super().__init__() self.dim = dim self.register_buffer('inv_freq', 1.0 / (10000 ** (torch.arange(0, dim, 2).float() / dim))) def forward(self, x): # x: (B, heads, S, head_dim) t = torch.arange(x.shape[-2], device=x.device, dtype=self.inv_freq.dtype) freqs = torch.outer(t, self.inv_freq.to(x.device)) emb = torch.cat([freqs, freqs], dim=-1) cos, sin = emb.cos()[None, None], emb.sin()[None, None] x1, x2 = x[..., : x.shape[-1] // 2], x[..., x.shape[-1] // 2:] return x * cos + torch.cat([-x2, x1], dim=-1) * sin class Mlp(nn.Module): """The ViT MLP (timm.layers.Mlp) as DextraVerse's nets/layers.py.""" def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.0): super().__init__() out_features = out_features or in_features hidden_features = hidden_features or in_features self.fc1 = nn.Linear(in_features, hidden_features) self.act = act_layer() self.drop1 = nn.Dropout(drop) self.norm = nn.Identity() self.fc2 = nn.Linear(hidden_features, out_features) self.drop2 = nn.Dropout(drop) def forward(self, x): return self.drop2(self.fc2(self.norm(self.drop1(self.act(self.fc1(x)))))) class Attention(nn.Module): def __init__(self, dim, num_heads=8, qkv_bias=True, attn_drop=0.0, proj_drop=0.0, qk_norm=True): super().__init__() assert dim % num_heads == 0, 'dim should be divisible by num_heads' self.num_heads = num_heads self.head_dim = dim // num_heads self.attn_drop = attn_drop self.proj_drop = nn.Dropout(proj_drop) self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) self.q_norm = nn.LayerNorm(self.head_dim) if qk_norm else nn.Identity() self.k_norm = nn.LayerNorm(self.head_dim) if qk_norm else nn.Identity() self.proj = nn.Linear(dim, dim) self.rope = RotaryPosEmb(self.head_dim) def forward(self, x): B, S, C = x.shape qkv = self.qkv(x).reshape(B, S, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4) q, k, v = qkv.unbind(0) q, k = self.rope(self.q_norm(q)), self.rope(self.k_norm(k)) x = F.scaled_dot_product_attention(q, k, v, dropout_p=self.attn_drop if self.training else 0.0) return self.proj_drop(self.proj(x.transpose(1, 2).reshape(B, S, C))) class AdaLNBlock(nn.Module): """Adaptive-LayerNorm transformer block as in DiT (Peebles et al. 2022).""" def __init__(self, dim, num_heads, mlp_ratio=4.0, dropout=0.0): super().__init__() self.norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) self.attn = Attention(dim, num_heads=num_heads, attn_drop=dropout, proj_drop=dropout) self.norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) self.mlp = Mlp(in_features=dim, hidden_features=int(dim * mlp_ratio), act_layer=lambda: nn.GELU(approximate='tanh'), drop=0) self.ada_ln = nn.Sequential(nn.SiLU(), nn.Linear(dim, 6 * dim)) self.dim = dim nn.init.constant_(self.ada_ln[-1].weight, 0) nn.init.constant_(self.ada_ln[-1].bias, 0) def forward(self, x, t, c): B = x.shape[0] gamma1, gamma2, scale1, scale2, shift1, shift2 = self.ada_ln(F.silu(t + c)).view(B, 6, 1, self.dim).unbind(1) x = x + self.attn(self.norm1(x).mul(scale1.add(1)).add_(shift1)).mul_(gamma1) return x + self.mlp(self.norm2(x).mul(scale2.add(1)).add_(shift2)).mul_(gamma2) class FlowTransformer(nn.Module): """DiT action backbone: AdaLN blocks over the H action tokens, conditioned on time_embed(t) + cond_embed(obs). As DextraVerse, the generic Xavier initialization runs after the blocks' zero init of their AdaLN projection, so it is not the full AdaLN-Zero recipe (kept: it is the reference's training behaviour).""" def __init__(self, input_dim, condition_dim, hidden_dim, output_dim, num_layers, num_heads, mlp_ratio=4.0, dropout=0.1, time_embed_dim=256): super().__init__() self.input_proj = nn.Linear(input_dim, hidden_dim) self.time_embed = nn.Sequential(SinusoidalPosEmb(time_embed_dim), nn.Linear(time_embed_dim, time_embed_dim * 4), nn.Mish(), nn.Linear(time_embed_dim * 4, hidden_dim)) self.cond_embed = nn.Linear(condition_dim, hidden_dim) self.transformer_blocks = nn.ModuleList( [AdaLNBlock(hidden_dim, num_heads, mlp_ratio=mlp_ratio, dropout=dropout) for _ in range(num_layers)]) self.norm = nn.LayerNorm(hidden_dim) self.out_proj = nn.Linear(hidden_dim, output_dim) def basic_init(module): if isinstance(module, nn.Linear): nn.init.xavier_uniform_(module.weight) if module.bias is not None: nn.init.constant_(module.bias, 0) self.apply(basic_init) nn.init.normal_(self.time_embed[1].weight, std=0.02) nn.init.normal_(self.time_embed[3].weight, std=0.02) def forward(self, x, t, global_cond): if not torch.is_tensor(t): t = torch.tensor(t, device=x.device) if t.ndim == 0: t = t.expand(x.shape[0]) x = self.input_proj(x) t = self.time_embed(t) c = self.cond_embed(global_cond) for block in self.transformer_blocks: x = block(x, t, c) return self.out_proj(self.norm(x)) def replace_batchnorm_with_groupnorm(module: nn.Module) -> nn.Module: """DextraVerse's use_group_norm: every BatchNorm2d becomes GroupNorm(num_features // 16 groups).""" for name, child in module.named_children(): if isinstance(child, nn.BatchNorm2d): setattr(module, name, nn.GroupNorm(num_groups=child.num_features // 16, num_channels=child.num_features)) else: replace_batchnorm_with_groupnorm(child) return module NORMS = ('group', 'frozen_bn') def get_resnet(name='resnet18', weights=None, norm='group') -> nn.Module: """A torchvision ResNet with its classifier removed: (N, 3, H, W) -> (N, 512) globally pooled. weights: torchvision weights name (e.g. 'IMAGENET1K_V1') to initialize from, or None (random initialization; a checkpoint load supplies every weight). norm: 'group' replaces every BatchNorm with GroupNorm (DextraVerse's use_group_norm, for training from scratch: pretrained BatchNorm statistics and affine are dropped); 'frozen_bn' keeps BatchNorm frozen at its (pretrained) running statistics and affine, the convolutions fine-tune.""" if norm == 'frozen_bn': from .act_nets import FrozenBatchNorm2d resnet = getattr(torchvision.models, name)(weights=weights, norm_layer=FrozenBatchNorm2d) elif norm == 'group': resnet = replace_batchnorm_with_groupnorm(getattr(torchvision.models, name)(weights=weights)) else: raise ValueError(f'norm must be one of {NORMS}, got {norm!r}') resnet.fc = nn.Identity() return resnet class ImageStateEncoder(nn.Module): """DextraVerse MultiImageObsEncoder as configured for FM-DiT / DDPM-DiT: one ResNet shared by every camera (share_rgb_model; `get_resnet` weights and norm), ImageNet normalization of the [0, 1] images, the pooled features of each camera in the contract's camera order, then the normalized state. Output (B, n_cameras * 512 + state_dim).""" def __init__(self, n_cameras, state_dim, backbone='resnet18', weights=None, norm='group'): super().__init__() self.rgb = get_resnet(backbone, weights=weights, norm=norm) self.n_cameras, self.state_dim = int(n_cameras), int(state_dim) self.register_buffer('mean', torch.tensor(IMAGENET_MEAN).view(1, 3, 1, 1), persistent=False) self.register_buffer('std', torch.tensor(IMAGENET_STD).view(1, 3, 1, 1), persistent=False) self.feature_dim = self.n_cameras * (512 if backbone in ('resnet18', 'resnet34') else 2048) + self.state_dim def forward(self, images, state): """images (B, n_cameras, 3, H, W) in [0, 1]; state (B, state_dim) normalized.""" B = images.shape[0] x = (images.flatten(0, 1) - self.mean) / self.std features = self.rgb(x).reshape(B, -1) # camera features side by side, in camera order return torch.cat([features, state], dim=-1)