ZUO66's picture
Add A20 batch-512 trim-v2 step-4000 checkpoint
3e7c150 verified
Raw History Blame Contribute Delete
9.99 kB
"""Networks of the FM-DiT and DP (DDPM-DiT) baselines: the AdaLN DiT action backbone and the camera + state encoder.
Ported from DextraVerse's imitation-learning core (`dextraverse/algorithms/il/models/nets/dit.py`, `layers.py`,
`positional_embedding.py`, `vision/backbones.py`, `vision/encoder.py`), unchanged in architecture and initialization.
Left out: the ROVE improvement embedding, the spatial-softmax / R3M / tactile options and the inpainting path (the
baselines always condition through the global observation feature). Added: the ResNet's normalization choice
(`get_resnet`): DextraVerse's GroupNorm for a backbone trained from scratch, or BatchNorm frozen at the pretrained
statistics (the DETR / ACT practice) for an ImageNet-initialized backbone that is fine-tuned with the policy.
"""
from __future__ import annotations
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision
IMAGENET_MEAN = (0.485, 0.456, 0.406)
IMAGENET_STD = (0.229, 0.224, 0.225)
class SinusoidalPosEmb(nn.Module):
def __init__(self, dim):
super().__init__()
self.dim = dim
def forward(self, x):
half_dim = self.dim // 2
emb = math.log(10000) / (half_dim - 1)
emb = torch.exp(torch.arange(half_dim, device=x.device) * -emb)
emb = x[:, None] * emb[None, :]
return torch.cat((emb.sin(), emb.cos()), dim=-1)
class RotaryPosEmb(nn.Module):
"""Rotary position embedding over each attention head's (S, head_dim) slice."""
def __init__(self, dim):
super().__init__()
self.dim = dim
self.register_buffer('inv_freq', 1.0 / (10000 ** (torch.arange(0, dim, 2).float() / dim)))
def forward(self, x): # x: (B, heads, S, head_dim)
t = torch.arange(x.shape[-2], device=x.device, dtype=self.inv_freq.dtype)
freqs = torch.outer(t, self.inv_freq.to(x.device))
emb = torch.cat([freqs, freqs], dim=-1)
cos, sin = emb.cos()[None, None], emb.sin()[None, None]
x1, x2 = x[..., : x.shape[-1] // 2], x[..., x.shape[-1] // 2:]
return x * cos + torch.cat([-x2, x1], dim=-1) * sin
class Mlp(nn.Module):
"""The ViT MLP (timm.layers.Mlp) as DextraVerse's nets/layers.py."""
def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.0):
super().__init__()
out_features = out_features or in_features
hidden_features = hidden_features or in_features
self.fc1 = nn.Linear(in_features, hidden_features)
self.act = act_layer()
self.drop1 = nn.Dropout(drop)
self.norm = nn.Identity()
self.fc2 = nn.Linear(hidden_features, out_features)
self.drop2 = nn.Dropout(drop)
def forward(self, x):
return self.drop2(self.fc2(self.norm(self.drop1(self.act(self.fc1(x))))))
class Attention(nn.Module):
def __init__(self, dim, num_heads=8, qkv_bias=True, attn_drop=0.0, proj_drop=0.0, qk_norm=True):
super().__init__()
assert dim % num_heads == 0, 'dim should be divisible by num_heads'
self.num_heads = num_heads
self.head_dim = dim // num_heads
self.attn_drop = attn_drop
self.proj_drop = nn.Dropout(proj_drop)
self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
self.q_norm = nn.LayerNorm(self.head_dim) if qk_norm else nn.Identity()
self.k_norm = nn.LayerNorm(self.head_dim) if qk_norm else nn.Identity()
self.proj = nn.Linear(dim, dim)
self.rope = RotaryPosEmb(self.head_dim)
def forward(self, x):
B, S, C = x.shape
qkv = self.qkv(x).reshape(B, S, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4)
q, k, v = qkv.unbind(0)
q, k = self.rope(self.q_norm(q)), self.rope(self.k_norm(k))
x = F.scaled_dot_product_attention(q, k, v, dropout_p=self.attn_drop if self.training else 0.0)
return self.proj_drop(self.proj(x.transpose(1, 2).reshape(B, S, C)))
class AdaLNBlock(nn.Module):
"""Adaptive-LayerNorm transformer block as in DiT (Peebles et al. 2022)."""
def __init__(self, dim, num_heads, mlp_ratio=4.0, dropout=0.0):
super().__init__()
self.norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
self.attn = Attention(dim, num_heads=num_heads, attn_drop=dropout, proj_drop=dropout)
self.norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
self.mlp = Mlp(in_features=dim, hidden_features=int(dim * mlp_ratio),
act_layer=lambda: nn.GELU(approximate='tanh'), drop=0)
self.ada_ln = nn.Sequential(nn.SiLU(), nn.Linear(dim, 6 * dim))
self.dim = dim
nn.init.constant_(self.ada_ln[-1].weight, 0)
nn.init.constant_(self.ada_ln[-1].bias, 0)
def forward(self, x, t, c):
B = x.shape[0]
gamma1, gamma2, scale1, scale2, shift1, shift2 = self.ada_ln(F.silu(t + c)).view(B, 6, 1, self.dim).unbind(1)
x = x + self.attn(self.norm1(x).mul(scale1.add(1)).add_(shift1)).mul_(gamma1)
return x + self.mlp(self.norm2(x).mul(scale2.add(1)).add_(shift2)).mul_(gamma2)
class FlowTransformer(nn.Module):
"""DiT action backbone: AdaLN blocks over the H action tokens, conditioned on time_embed(t) + cond_embed(obs).
As DextraVerse, the generic Xavier initialization runs after the blocks' zero init of their AdaLN projection, so
it is not the full AdaLN-Zero recipe (kept: it is the reference's training behaviour)."""
def __init__(self, input_dim, condition_dim, hidden_dim, output_dim, num_layers, num_heads, mlp_ratio=4.0,
dropout=0.1, time_embed_dim=256):
super().__init__()
self.input_proj = nn.Linear(input_dim, hidden_dim)
self.time_embed = nn.Sequential(SinusoidalPosEmb(time_embed_dim), nn.Linear(time_embed_dim, time_embed_dim * 4),
nn.Mish(), nn.Linear(time_embed_dim * 4, hidden_dim))
self.cond_embed = nn.Linear(condition_dim, hidden_dim)
self.transformer_blocks = nn.ModuleList(
[AdaLNBlock(hidden_dim, num_heads, mlp_ratio=mlp_ratio, dropout=dropout) for _ in range(num_layers)])
self.norm = nn.LayerNorm(hidden_dim)
self.out_proj = nn.Linear(hidden_dim, output_dim)
def basic_init(module):
if isinstance(module, nn.Linear):
nn.init.xavier_uniform_(module.weight)
if module.bias is not None:
nn.init.constant_(module.bias, 0)
self.apply(basic_init)
nn.init.normal_(self.time_embed[1].weight, std=0.02)
nn.init.normal_(self.time_embed[3].weight, std=0.02)
def forward(self, x, t, global_cond):
if not torch.is_tensor(t):
t = torch.tensor(t, device=x.device)
if t.ndim == 0:
t = t.expand(x.shape[0])
x = self.input_proj(x)
t = self.time_embed(t)
c = self.cond_embed(global_cond)
for block in self.transformer_blocks:
x = block(x, t, c)
return self.out_proj(self.norm(x))
def replace_batchnorm_with_groupnorm(module: nn.Module) -> nn.Module:
"""DextraVerse's use_group_norm: every BatchNorm2d becomes GroupNorm(num_features // 16 groups)."""
for name, child in module.named_children():
if isinstance(child, nn.BatchNorm2d):
setattr(module, name, nn.GroupNorm(num_groups=child.num_features // 16, num_channels=child.num_features))
else:
replace_batchnorm_with_groupnorm(child)
return module
NORMS = ('group', 'frozen_bn')
def get_resnet(name='resnet18', weights=None, norm='group') -> nn.Module:
"""A torchvision ResNet with its classifier removed: (N, 3, H, W) -> (N, 512) globally pooled.
weights: torchvision weights name (e.g. 'IMAGENET1K_V1') to initialize from, or None (random initialization;
a checkpoint load supplies every weight). norm: 'group' replaces every BatchNorm with GroupNorm (DextraVerse's
use_group_norm, for training from scratch: pretrained BatchNorm statistics and affine are dropped); 'frozen_bn'
keeps BatchNorm frozen at its (pretrained) running statistics and affine, the convolutions fine-tune."""
if norm == 'frozen_bn':
from .act_nets import FrozenBatchNorm2d
resnet = getattr(torchvision.models, name)(weights=weights, norm_layer=FrozenBatchNorm2d)
elif norm == 'group':
resnet = replace_batchnorm_with_groupnorm(getattr(torchvision.models, name)(weights=weights))
else:
raise ValueError(f'norm must be one of {NORMS}, got {norm!r}')
resnet.fc = nn.Identity()
return resnet
class ImageStateEncoder(nn.Module):
"""DextraVerse MultiImageObsEncoder as configured for FM-DiT / DDPM-DiT: one ResNet shared by every camera
(share_rgb_model; `get_resnet` weights and norm), ImageNet normalization of the [0, 1] images, the pooled features
of each camera in the contract's camera order, then the normalized state.
Output (B, n_cameras * 512 + state_dim)."""
def __init__(self, n_cameras, state_dim, backbone='resnet18', weights=None, norm='group'):
super().__init__()
self.rgb = get_resnet(backbone, weights=weights, norm=norm)
self.n_cameras, self.state_dim = int(n_cameras), int(state_dim)
self.register_buffer('mean', torch.tensor(IMAGENET_MEAN).view(1, 3, 1, 1), persistent=False)
self.register_buffer('std', torch.tensor(IMAGENET_STD).view(1, 3, 1, 1), persistent=False)
self.feature_dim = self.n_cameras * (512 if backbone in ('resnet18', 'resnet34') else 2048) + self.state_dim
def forward(self, images, state):
"""images (B, n_cameras, 3, H, W) in [0, 1]; state (B, state_dim) normalized."""
B = images.shape[0]
x = (images.flatten(0, 1) - self.mean) / self.std
features = self.rgb(x).reshape(B, -1) # camera features side by side, in camera order
return torch.cat([features, state], dim=-1)