"""ACT (Zhao et al. 2023, "Learning Fine-Grained Bimanual Manipulation with Low-Cost Hardware") network core: the ResNet backbone with frozen BatchNorm and sine position embedding, the DETR encoder-decoder transformer and the CVAE action-chunk decoder. Ported unchanged from DextraVerse (`dextraverse/algorithms/il/models/nets/act/`), which vendors the ACT repository's `detr/models/{backbone,position_encoding,transformer,detr_vae}.py` (Apache-2.0, (c) Facebook, Inc. and its affiliates; see THIRD_PARTY_NOTICES.md) with the NestedTensor/argparse plumbing removed, explicit `qpos_dim` / `action_dim` instead of the ALOHA 14-DoF constants, and one backbone shared by every camera (upstream behaviour). The policy uses `return_intermediate_dec=False`: `Transformer.forward(...)[0]` is the final decoder layer. """ from __future__ import annotations import copy import math from typing import Optional import numpy as np import torch import torch.nn.functional as F import torchvision from torch import Tensor, nn from torchvision.models._utils import IntermediateLayerGetter class PositionEmbeddingSine(nn.Module): """Standard DETR sine embedding over a ``(B, C, H, W)`` feature map.""" def __init__( self, num_pos_feats: int = 64, temperature: int = 10000, normalize: bool = False ) -> None: super().__init__() self.num_pos_feats = num_pos_feats self.temperature = temperature self.normalize = normalize self.scale = 2 * math.pi def forward(self, x: torch.Tensor) -> torch.Tensor: not_mask = torch.ones((1, x.shape[-2], x.shape[-1]), device=x.device) y_embed = not_mask.cumsum(1, dtype=torch.float32) x_embed = not_mask.cumsum(2, dtype=torch.float32) if self.normalize: eps = 1e-6 y_embed = y_embed / (y_embed[:, -1:, :] + eps) * self.scale x_embed = x_embed / (x_embed[:, :, -1:] + eps) * self.scale dim_t = torch.arange(self.num_pos_feats, dtype=torch.float32, device=x.device) dim_t = self.temperature ** (2 * (dim_t // 2) / self.num_pos_feats) pos_x = x_embed[:, :, :, None] / dim_t pos_y = y_embed[:, :, :, None] / dim_t pos_x = torch.stack( (pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4 ).flatten(3) pos_y = torch.stack( (pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4 ).flatten(3) return torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2) class FrozenBatchNorm2d(nn.Module): """BatchNorm2d with fixed statistics and affine parameters (the DETR/ACT backbone norm).""" def __init__(self, n: int) -> None: super().__init__() self.register_buffer("weight", torch.ones(n)) self.register_buffer("bias", torch.zeros(n)) self.register_buffer("running_mean", torch.zeros(n)) self.register_buffer("running_var", torch.ones(n)) def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, *args): state_dict.pop(prefix + "num_batches_tracked", None) super()._load_from_state_dict(state_dict, prefix, local_metadata, strict, *args) def forward(self, x: torch.Tensor) -> torch.Tensor: w = self.weight.reshape(1, -1, 1, 1) b = self.bias.reshape(1, -1, 1, 1) rv = self.running_var.reshape(1, -1, 1, 1) rm = self.running_mean.reshape(1, -1, 1, 1) scale = w * (rv + 1e-5).rsqrt() return x * scale + (b - rm * scale) class Backbone(nn.Module): """ResNet trunk returning the layer4 feature map.""" def __init__(self, name: str = "resnet18", weights: str | None = None) -> None: super().__init__() trunk = getattr(torchvision.models, name)( replace_stride_with_dilation=[False, False, False], weights=weights, norm_layer=FrozenBatchNorm2d, ) self.body = IntermediateLayerGetter(trunk, return_layers={"layer4": "0"}) self.num_channels = 512 if name in ("resnet18", "resnet34") else 2048 def forward(self, x: torch.Tensor) -> dict[str, torch.Tensor]: return self.body(x) class Joiner(nn.Sequential): """Backbone + position embedding, returning matched feature/position lists.""" def __init__(self, backbone: Backbone, position_embedding: PositionEmbeddingSine) -> None: super().__init__(backbone, position_embedding) self.num_channels = backbone.num_channels def forward(self, x: torch.Tensor) -> tuple[list[torch.Tensor], list[torch.Tensor]]: features = list(self[0](x).values()) pos = [self[1](f).to(f.dtype) for f in features] return features, pos def build_backbone( name: str = "resnet18", hidden_dim: int = 512, weights: str | None = None ) -> Joiner: """The ACT camera backbone: ResNet + sine embedding sized to the transformer width.""" return Joiner(Backbone(name, weights=weights), PositionEmbeddingSine(hidden_dim // 2)) class Transformer(nn.Module): def __init__( self, d_model=512, nhead=8, num_encoder_layers=6, num_decoder_layers=6, dim_feedforward=2048, dropout=0.1, activation="relu", normalize_before=False, return_intermediate_dec=False, ): super().__init__() encoder_layer = TransformerEncoderLayer( d_model, nhead, dim_feedforward, dropout, activation, normalize_before ) encoder_norm = nn.LayerNorm(d_model) if normalize_before else None self.encoder = TransformerEncoder(encoder_layer, num_encoder_layers, encoder_norm) decoder_layer = TransformerDecoderLayer( d_model, nhead, dim_feedforward, dropout, activation, normalize_before ) decoder_norm = nn.LayerNorm(d_model) self.decoder = TransformerDecoder( decoder_layer, num_decoder_layers, decoder_norm, return_intermediate=return_intermediate_dec, ) self._reset_parameters() self.d_model = d_model self.nhead = nhead def _reset_parameters(self): for p in self.parameters(): if p.dim() > 1: nn.init.xavier_uniform_(p) def forward( self, src, mask, query_embed, pos_embed, latent_input=None, proprio_input=None, additional_pos_embed=None, ): # TODO flatten only when input has H and W if len(src.shape) == 4: # has H and W # flatten NxCxHxW to HWxNxC bs, c, h, w = src.shape src = src.flatten(2).permute(2, 0, 1) pos_embed = pos_embed.flatten(2).permute(2, 0, 1).repeat(1, bs, 1) query_embed = query_embed.unsqueeze(1).repeat(1, bs, 1) # mask = mask.flatten(1) additional_pos_embed = additional_pos_embed.unsqueeze(1).repeat( 1, bs, 1 ) # seq, bs, dim pos_embed = torch.cat([additional_pos_embed, pos_embed], axis=0) addition_input = torch.stack([latent_input, proprio_input], axis=0) src = torch.cat([addition_input, src], axis=0) else: assert len(src.shape) == 3 # flatten NxHWxC to HWxNxC bs, hw, c = src.shape src = src.permute(1, 0, 2) pos_embed = pos_embed.unsqueeze(1).repeat(1, bs, 1) query_embed = query_embed.unsqueeze(1).repeat(1, bs, 1) tgt = torch.zeros_like(query_embed) memory = self.encoder(src, src_key_padding_mask=mask, pos=pos_embed) hs = self.decoder( tgt, memory, memory_key_padding_mask=mask, pos=pos_embed, query_pos=query_embed ) hs = hs.transpose(1, 2) return hs class TransformerEncoder(nn.Module): def __init__(self, encoder_layer, num_layers, norm=None): super().__init__() self.layers = _get_clones(encoder_layer, num_layers) self.num_layers = num_layers self.norm = norm def forward( self, src, mask: Optional[Tensor] = None, src_key_padding_mask: Optional[Tensor] = None, pos: Optional[Tensor] = None, ): output = src for layer in self.layers: output = layer( output, src_mask=mask, src_key_padding_mask=src_key_padding_mask, pos=pos ) if self.norm is not None: output = self.norm(output) return output class TransformerDecoder(nn.Module): def __init__(self, decoder_layer, num_layers, norm=None, return_intermediate=False): super().__init__() self.layers = _get_clones(decoder_layer, num_layers) self.num_layers = num_layers self.norm = norm self.return_intermediate = return_intermediate def forward( self, tgt, memory, tgt_mask: Optional[Tensor] = None, memory_mask: Optional[Tensor] = None, tgt_key_padding_mask: Optional[Tensor] = None, memory_key_padding_mask: Optional[Tensor] = None, pos: Optional[Tensor] = None, query_pos: Optional[Tensor] = None, ): output = tgt intermediate = [] for layer in self.layers: output = layer( output, memory, tgt_mask=tgt_mask, memory_mask=memory_mask, tgt_key_padding_mask=tgt_key_padding_mask, memory_key_padding_mask=memory_key_padding_mask, pos=pos, query_pos=query_pos, ) if self.return_intermediate: intermediate.append(self.norm(output)) if self.norm is not None: output = self.norm(output) if self.return_intermediate: intermediate.pop() intermediate.append(output) if self.return_intermediate: return torch.stack(intermediate) return output.unsqueeze(0) class TransformerEncoderLayer(nn.Module): def __init__( self, d_model, nhead, dim_feedforward=2048, dropout=0.1, activation="relu", normalize_before=False, ): super().__init__() self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout) # Implementation of Feedforward model self.linear1 = nn.Linear(d_model, dim_feedforward) self.dropout = nn.Dropout(dropout) self.linear2 = nn.Linear(dim_feedforward, d_model) self.norm1 = nn.LayerNorm(d_model) self.norm2 = nn.LayerNorm(d_model) self.dropout1 = nn.Dropout(dropout) self.dropout2 = nn.Dropout(dropout) self.activation = _get_activation_fn(activation) self.normalize_before = normalize_before def with_pos_embed(self, tensor, pos: Optional[Tensor]): return tensor if pos is None else tensor + pos def forward_post( self, src, src_mask: Optional[Tensor] = None, src_key_padding_mask: Optional[Tensor] = None, pos: Optional[Tensor] = None, ): q = k = self.with_pos_embed(src, pos) src2 = self.self_attn( q, k, value=src, attn_mask=src_mask, key_padding_mask=src_key_padding_mask )[0] src = src + self.dropout1(src2) src = self.norm1(src) src2 = self.linear2(self.dropout(self.activation(self.linear1(src)))) src = src + self.dropout2(src2) src = self.norm2(src) return src def forward_pre( self, src, src_mask: Optional[Tensor] = None, src_key_padding_mask: Optional[Tensor] = None, pos: Optional[Tensor] = None, ): src2 = self.norm1(src) q = k = self.with_pos_embed(src2, pos) src2 = self.self_attn( q, k, value=src2, attn_mask=src_mask, key_padding_mask=src_key_padding_mask )[0] src = src + self.dropout1(src2) src2 = self.norm2(src) src2 = self.linear2(self.dropout(self.activation(self.linear1(src2)))) src = src + self.dropout2(src2) return src def forward( self, src, src_mask: Optional[Tensor] = None, src_key_padding_mask: Optional[Tensor] = None, pos: Optional[Tensor] = None, ): if self.normalize_before: return self.forward_pre(src, src_mask, src_key_padding_mask, pos) return self.forward_post(src, src_mask, src_key_padding_mask, pos) class TransformerDecoderLayer(nn.Module): def __init__( self, d_model, nhead, dim_feedforward=2048, dropout=0.1, activation="relu", normalize_before=False, ): super().__init__() self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout) self.multihead_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout) # Implementation of Feedforward model self.linear1 = nn.Linear(d_model, dim_feedforward) self.dropout = nn.Dropout(dropout) self.linear2 = nn.Linear(dim_feedforward, d_model) self.norm1 = nn.LayerNorm(d_model) self.norm2 = nn.LayerNorm(d_model) self.norm3 = nn.LayerNorm(d_model) self.dropout1 = nn.Dropout(dropout) self.dropout2 = nn.Dropout(dropout) self.dropout3 = nn.Dropout(dropout) self.activation = _get_activation_fn(activation) self.normalize_before = normalize_before def with_pos_embed(self, tensor, pos: Optional[Tensor]): return tensor if pos is None else tensor + pos def forward_post( self, tgt, memory, tgt_mask: Optional[Tensor] = None, memory_mask: Optional[Tensor] = None, tgt_key_padding_mask: Optional[Tensor] = None, memory_key_padding_mask: Optional[Tensor] = None, pos: Optional[Tensor] = None, query_pos: Optional[Tensor] = None, ): q = k = self.with_pos_embed(tgt, query_pos) tgt2 = self.self_attn( q, k, value=tgt, attn_mask=tgt_mask, key_padding_mask=tgt_key_padding_mask )[0] tgt = tgt + self.dropout1(tgt2) tgt = self.norm1(tgt) tgt2 = self.multihead_attn( query=self.with_pos_embed(tgt, query_pos), key=self.with_pos_embed(memory, pos), value=memory, attn_mask=memory_mask, key_padding_mask=memory_key_padding_mask, )[0] tgt = tgt + self.dropout2(tgt2) tgt = self.norm2(tgt) tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt)))) tgt = tgt + self.dropout3(tgt2) tgt = self.norm3(tgt) return tgt def forward_pre( self, tgt, memory, tgt_mask: Optional[Tensor] = None, memory_mask: Optional[Tensor] = None, tgt_key_padding_mask: Optional[Tensor] = None, memory_key_padding_mask: Optional[Tensor] = None, pos: Optional[Tensor] = None, query_pos: Optional[Tensor] = None, ): tgt2 = self.norm1(tgt) q = k = self.with_pos_embed(tgt2, query_pos) tgt2 = self.self_attn( q, k, value=tgt2, attn_mask=tgt_mask, key_padding_mask=tgt_key_padding_mask )[0] tgt = tgt + self.dropout1(tgt2) tgt2 = self.norm2(tgt) tgt2 = self.multihead_attn( query=self.with_pos_embed(tgt2, query_pos), key=self.with_pos_embed(memory, pos), value=memory, attn_mask=memory_mask, key_padding_mask=memory_key_padding_mask, )[0] tgt = tgt + self.dropout2(tgt2) tgt2 = self.norm3(tgt) tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt2)))) tgt = tgt + self.dropout3(tgt2) return tgt def forward( self, tgt, memory, tgt_mask: Optional[Tensor] = None, memory_mask: Optional[Tensor] = None, tgt_key_padding_mask: Optional[Tensor] = None, memory_key_padding_mask: Optional[Tensor] = None, pos: Optional[Tensor] = None, query_pos: Optional[Tensor] = None, ): if self.normalize_before: return self.forward_pre( tgt, memory, tgt_mask, memory_mask, tgt_key_padding_mask, memory_key_padding_mask, pos, query_pos, ) return self.forward_post( tgt, memory, tgt_mask, memory_mask, tgt_key_padding_mask, memory_key_padding_mask, pos, query_pos, ) def _get_clones(module, N): return nn.ModuleList([copy.deepcopy(module) for i in range(N)]) def _get_activation_fn(activation): """Return an activation function given a string.""" if activation == "relu": return F.relu if activation == "gelu": return F.gelu raise RuntimeError(f"activation should be relu/gelu, not {activation}.") def reparametrize(mu: torch.Tensor, logvar: torch.Tensor) -> torch.Tensor: std = logvar.div(2).exp() return mu + std * torch.randn_like(std) def get_sinusoid_encoding_table(n_position: int, d_hid: int) -> torch.Tensor: def angle_vec(position: int) -> list[float]: return [position / np.power(10000, 2 * (j // 2) / d_hid) for j in range(d_hid)] table = np.array([angle_vec(i) for i in range(n_position)]) table[:, 0::2] = np.sin(table[:, 0::2]) table[:, 1::2] = np.cos(table[:, 1::2]) return torch.FloatTensor(table).unsqueeze(0) def build_style_encoder( hidden_dim: int, nheads: int, dim_feedforward: int, enc_layers: int, dropout: float ) -> TransformerEncoder: """The CVAE style encoder (a plain transformer encoder over [CLS, qpos, action seq]).""" layer = TransformerEncoderLayer(hidden_dim, nheads, dim_feedforward, dropout, "relu", False) return TransformerEncoder(layer, enc_layers, norm=None) class DETRVAE(nn.Module): """CVAE decoder: (qpos, images[, action seq]) → ``num_queries`` future actions.""" def __init__( self, backbone: Joiner, transformer: Transformer, encoder: TransformerEncoder, *, qpos_dim: int, action_dim: int, num_queries: int, latent_dim: int = 32, ) -> None: super().__init__() self.num_queries = num_queries self.transformer = transformer self.encoder = encoder hidden_dim = transformer.d_model self.action_head = nn.Linear(hidden_dim, action_dim) self.query_embed = nn.Embedding(num_queries, hidden_dim) self.backbone = backbone # ONE backbone shared by every camera (upstream behavior) self.input_proj = nn.Conv2d(backbone.num_channels, hidden_dim, kernel_size=1) self.input_proj_robot_state = nn.Linear(qpos_dim, hidden_dim) # CVAE style encoder inputs self.latent_dim = latent_dim self.cls_embed = nn.Embedding(1, hidden_dim) self.encoder_action_proj = nn.Linear(action_dim, hidden_dim) self.encoder_joint_proj = nn.Linear(qpos_dim, hidden_dim) self.latent_proj = nn.Linear(hidden_dim, latent_dim * 2) self.register_buffer( "pos_table", get_sinusoid_encoding_table(1 + 1 + num_queries, hidden_dim) ) # decoder side self.latent_out_proj = nn.Linear(latent_dim, hidden_dim) self.additional_pos_embed = nn.Embedding(2, hidden_dim) # latent + proprio slots def forward( self, qpos: torch.Tensor, images: torch.Tensor, actions: torch.Tensor | None = None, is_pad: torch.Tensor | None = None, ) -> tuple[torch.Tensor, tuple[torch.Tensor | None, torch.Tensor | None]]: """``qpos (B, Dq)``, ``images (B, ncam, C, H, W)``; train passes ``actions (B, T, Da)``.""" is_training = actions is not None bs = qpos.shape[0] if is_training: action_embed = self.encoder_action_proj(actions) qpos_embed = self.encoder_joint_proj(qpos).unsqueeze(1) cls_embed = self.cls_embed.weight.unsqueeze(0).repeat(bs, 1, 1) encoder_input = torch.cat([cls_embed, qpos_embed, action_embed], dim=1) encoder_input = encoder_input.permute(1, 0, 2) if is_pad is None: is_pad = torch.zeros(bs, actions.shape[1], dtype=torch.bool, device=qpos.device) cls_joint_is_pad = torch.full((bs, 2), False, device=qpos.device) key_pad = torch.cat([cls_joint_is_pad, is_pad], dim=1) pos_embed = self.pos_table.clone().detach().permute(1, 0, 2) encoder_output = self.encoder( encoder_input, pos=pos_embed, src_key_padding_mask=key_pad )[0] latent_info = self.latent_proj(encoder_output) mu = latent_info[:, : self.latent_dim] logvar = latent_info[:, self.latent_dim :] latent_input = self.latent_out_proj(reparametrize(mu, logvar)) else: mu = logvar = None latent_sample = torch.zeros(bs, self.latent_dim, device=qpos.device) latent_input = self.latent_out_proj(latent_sample) all_cam_features, all_cam_pos = [], [] for cam_id in range(images.shape[1]): features, pos = self.backbone(images[:, cam_id]) all_cam_features.append(self.input_proj(features[0])) all_cam_pos.append(pos[0]) proprio_input = self.input_proj_robot_state(qpos) src = torch.cat(all_cam_features, dim=3) # cameras fold into the width axis pos = torch.cat(all_cam_pos, dim=3) hs = self.transformer( src, None, self.query_embed.weight, pos, latent_input, proprio_input, self.additional_pos_embed.weight, )[0] return self.action_head(hs), (mu, logvar) def kl_divergence(mu: torch.Tensor, logvar: torch.Tensor) -> torch.Tensor: """Total KL(q(z|x) || N(0, I)), summed over latent dims, meaned over the batch.""" klds = -0.5 * (1 + logvar - mu.pow(2) - logvar.exp()) return klds.sum(1).mean()