Download code/hb_baselines_v1/baselines/nets.py from humanoid-benchmark/dp: direct link, hf CLI and curl.
- Browser
- Download file 9.99 kB
-
https://huggingface.co/humanoid-benchmark/dp/resolve/main/code/hb_baselines_v1/baselines/nets.py
- Command line
-
hf download hf://humanoid-benchmark/dp/code/hb_baselines_v1/baselines/nets.py
-
curl -L -o nets.py https://huggingface.co/humanoid-benchmark/dp/resolve/main/code/hb_baselines_v1/baselines/nets.py
9.99 kB
| """Networks of the FM-DiT and DP (DDPM-DiT) baselines: the AdaLN DiT action backbone and the camera + state encoder. | |
| Ported from DextraVerse's imitation-learning core (`dextraverse/algorithms/il/models/nets/dit.py`, `layers.py`, | |
| `positional_embedding.py`, `vision/backbones.py`, `vision/encoder.py`), unchanged in architecture and initialization. | |
| Left out: the ROVE improvement embedding, the spatial-softmax / R3M / tactile options and the inpainting path (the | |
| baselines always condition through the global observation feature). Added: the ResNet's normalization choice | |
| (`get_resnet`): DextraVerse's GroupNorm for a backbone trained from scratch, or BatchNorm frozen at the pretrained | |
| statistics (the DETR / ACT practice) for an ImageNet-initialized backbone that is fine-tuned with the policy. | |
| """ | |
| from __future__ import annotations | |
| import math | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| import torchvision | |
| IMAGENET_MEAN = (0.485, 0.456, 0.406) | |
| IMAGENET_STD = (0.229, 0.224, 0.225) | |
| class SinusoidalPosEmb(nn.Module): | |
| def __init__(self, dim): | |
| super().__init__() | |
| self.dim = dim | |
| def forward(self, x): | |
| half_dim = self.dim // 2 | |
| emb = math.log(10000) / (half_dim - 1) | |
| emb = torch.exp(torch.arange(half_dim, device=x.device) * -emb) | |
| emb = x[:, None] * emb[None, :] | |
| return torch.cat((emb.sin(), emb.cos()), dim=-1) | |
| class RotaryPosEmb(nn.Module): | |
| """Rotary position embedding over each attention head's (S, head_dim) slice.""" | |
| def __init__(self, dim): | |
| super().__init__() | |
| self.dim = dim | |
| self.register_buffer('inv_freq', 1.0 / (10000 ** (torch.arange(0, dim, 2).float() / dim))) | |
| def forward(self, x): # x: (B, heads, S, head_dim) | |
| t = torch.arange(x.shape[-2], device=x.device, dtype=self.inv_freq.dtype) | |
| freqs = torch.outer(t, self.inv_freq.to(x.device)) | |
| emb = torch.cat([freqs, freqs], dim=-1) | |
| cos, sin = emb.cos()[None, None], emb.sin()[None, None] | |
| x1, x2 = x[..., : x.shape[-1] // 2], x[..., x.shape[-1] // 2:] | |
| return x * cos + torch.cat([-x2, x1], dim=-1) * sin | |
| class Mlp(nn.Module): | |
| """The ViT MLP (timm.layers.Mlp) as DextraVerse's nets/layers.py.""" | |
| def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.0): | |
| super().__init__() | |
| out_features = out_features or in_features | |
| hidden_features = hidden_features or in_features | |
| self.fc1 = nn.Linear(in_features, hidden_features) | |
| self.act = act_layer() | |
| self.drop1 = nn.Dropout(drop) | |
| self.norm = nn.Identity() | |
| self.fc2 = nn.Linear(hidden_features, out_features) | |
| self.drop2 = nn.Dropout(drop) | |
| def forward(self, x): | |
| return self.drop2(self.fc2(self.norm(self.drop1(self.act(self.fc1(x)))))) | |
| class Attention(nn.Module): | |
| def __init__(self, dim, num_heads=8, qkv_bias=True, attn_drop=0.0, proj_drop=0.0, qk_norm=True): | |
| super().__init__() | |
| assert dim % num_heads == 0, 'dim should be divisible by num_heads' | |
| self.num_heads = num_heads | |
| self.head_dim = dim // num_heads | |
| self.attn_drop = attn_drop | |
| self.proj_drop = nn.Dropout(proj_drop) | |
| self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) | |
| self.q_norm = nn.LayerNorm(self.head_dim) if qk_norm else nn.Identity() | |
| self.k_norm = nn.LayerNorm(self.head_dim) if qk_norm else nn.Identity() | |
| self.proj = nn.Linear(dim, dim) | |
| self.rope = RotaryPosEmb(self.head_dim) | |
| def forward(self, x): | |
| B, S, C = x.shape | |
| qkv = self.qkv(x).reshape(B, S, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4) | |
| q, k, v = qkv.unbind(0) | |
| q, k = self.rope(self.q_norm(q)), self.rope(self.k_norm(k)) | |
| x = F.scaled_dot_product_attention(q, k, v, dropout_p=self.attn_drop if self.training else 0.0) | |
| return self.proj_drop(self.proj(x.transpose(1, 2).reshape(B, S, C))) | |
| class AdaLNBlock(nn.Module): | |
| """Adaptive-LayerNorm transformer block as in DiT (Peebles et al. 2022).""" | |
| def __init__(self, dim, num_heads, mlp_ratio=4.0, dropout=0.0): | |
| super().__init__() | |
| self.norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) | |
| self.attn = Attention(dim, num_heads=num_heads, attn_drop=dropout, proj_drop=dropout) | |
| self.norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) | |
| self.mlp = Mlp(in_features=dim, hidden_features=int(dim * mlp_ratio), | |
| act_layer=lambda: nn.GELU(approximate='tanh'), drop=0) | |
| self.ada_ln = nn.Sequential(nn.SiLU(), nn.Linear(dim, 6 * dim)) | |
| self.dim = dim | |
| nn.init.constant_(self.ada_ln[-1].weight, 0) | |
| nn.init.constant_(self.ada_ln[-1].bias, 0) | |
| def forward(self, x, t, c): | |
| B = x.shape[0] | |
| gamma1, gamma2, scale1, scale2, shift1, shift2 = self.ada_ln(F.silu(t + c)).view(B, 6, 1, self.dim).unbind(1) | |
| x = x + self.attn(self.norm1(x).mul(scale1.add(1)).add_(shift1)).mul_(gamma1) | |
| return x + self.mlp(self.norm2(x).mul(scale2.add(1)).add_(shift2)).mul_(gamma2) | |
| class FlowTransformer(nn.Module): | |
| """DiT action backbone: AdaLN blocks over the H action tokens, conditioned on time_embed(t) + cond_embed(obs). | |
| As DextraVerse, the generic Xavier initialization runs after the blocks' zero init of their AdaLN projection, so | |
| it is not the full AdaLN-Zero recipe (kept: it is the reference's training behaviour).""" | |
| def __init__(self, input_dim, condition_dim, hidden_dim, output_dim, num_layers, num_heads, mlp_ratio=4.0, | |
| dropout=0.1, time_embed_dim=256): | |
| super().__init__() | |
| self.input_proj = nn.Linear(input_dim, hidden_dim) | |
| self.time_embed = nn.Sequential(SinusoidalPosEmb(time_embed_dim), nn.Linear(time_embed_dim, time_embed_dim * 4), | |
| nn.Mish(), nn.Linear(time_embed_dim * 4, hidden_dim)) | |
| self.cond_embed = nn.Linear(condition_dim, hidden_dim) | |
| self.transformer_blocks = nn.ModuleList( | |
| [AdaLNBlock(hidden_dim, num_heads, mlp_ratio=mlp_ratio, dropout=dropout) for _ in range(num_layers)]) | |
| self.norm = nn.LayerNorm(hidden_dim) | |
| self.out_proj = nn.Linear(hidden_dim, output_dim) | |
| def basic_init(module): | |
| if isinstance(module, nn.Linear): | |
| nn.init.xavier_uniform_(module.weight) | |
| if module.bias is not None: | |
| nn.init.constant_(module.bias, 0) | |
| self.apply(basic_init) | |
| nn.init.normal_(self.time_embed[1].weight, std=0.02) | |
| nn.init.normal_(self.time_embed[3].weight, std=0.02) | |
| def forward(self, x, t, global_cond): | |
| if not torch.is_tensor(t): | |
| t = torch.tensor(t, device=x.device) | |
| if t.ndim == 0: | |
| t = t.expand(x.shape[0]) | |
| x = self.input_proj(x) | |
| t = self.time_embed(t) | |
| c = self.cond_embed(global_cond) | |
| for block in self.transformer_blocks: | |
| x = block(x, t, c) | |
| return self.out_proj(self.norm(x)) | |
| def replace_batchnorm_with_groupnorm(module: nn.Module) -> nn.Module: | |
| """DextraVerse's use_group_norm: every BatchNorm2d becomes GroupNorm(num_features // 16 groups).""" | |
| for name, child in module.named_children(): | |
| if isinstance(child, nn.BatchNorm2d): | |
| setattr(module, name, nn.GroupNorm(num_groups=child.num_features // 16, num_channels=child.num_features)) | |
| else: | |
| replace_batchnorm_with_groupnorm(child) | |
| return module | |
| NORMS = ('group', 'frozen_bn') | |
| def get_resnet(name='resnet18', weights=None, norm='group') -> nn.Module: | |
| """A torchvision ResNet with its classifier removed: (N, 3, H, W) -> (N, 512) globally pooled. | |
| weights: torchvision weights name (e.g. 'IMAGENET1K_V1') to initialize from, or None (random initialization; | |
| a checkpoint load supplies every weight). norm: 'group' replaces every BatchNorm with GroupNorm (DextraVerse's | |
| use_group_norm, for training from scratch: pretrained BatchNorm statistics and affine are dropped); 'frozen_bn' | |
| keeps BatchNorm frozen at its (pretrained) running statistics and affine, the convolutions fine-tune.""" | |
| if norm == 'frozen_bn': | |
| from .act_nets import FrozenBatchNorm2d | |
| resnet = getattr(torchvision.models, name)(weights=weights, norm_layer=FrozenBatchNorm2d) | |
| elif norm == 'group': | |
| resnet = replace_batchnorm_with_groupnorm(getattr(torchvision.models, name)(weights=weights)) | |
| else: | |
| raise ValueError(f'norm must be one of {NORMS}, got {norm!r}') | |
| resnet.fc = nn.Identity() | |
| return resnet | |
| class ImageStateEncoder(nn.Module): | |
| """DextraVerse MultiImageObsEncoder as configured for FM-DiT / DDPM-DiT: one ResNet shared by every camera | |
| (share_rgb_model; `get_resnet` weights and norm), ImageNet normalization of the [0, 1] images, the pooled features | |
| of each camera in the contract's camera order, then the normalized state. | |
| Output (B, n_cameras * 512 + state_dim).""" | |
| def __init__(self, n_cameras, state_dim, backbone='resnet18', weights=None, norm='group'): | |
| super().__init__() | |
| self.rgb = get_resnet(backbone, weights=weights, norm=norm) | |
| self.n_cameras, self.state_dim = int(n_cameras), int(state_dim) | |
| self.register_buffer('mean', torch.tensor(IMAGENET_MEAN).view(1, 3, 1, 1), persistent=False) | |
| self.register_buffer('std', torch.tensor(IMAGENET_STD).view(1, 3, 1, 1), persistent=False) | |
| self.feature_dim = self.n_cameras * (512 if backbone in ('resnet18', 'resnet34') else 2048) + self.state_dim | |
| def forward(self, images, state): | |
| """images (B, n_cameras, 3, H, W) in [0, 1]; state (B, state_dim) normalized.""" | |
| B = images.shape[0] | |
| x = (images.flatten(0, 1) - self.mean) / self.std | |
| features = self.rgb(x).reshape(B, -1) # camera features side by side, in camera order | |
| return torch.cat([features, state], dim=-1) | |