dp / code /hb_baselines_v1 /baselines /act_nets.py
BIue-Jay's picture
Add B6 DP batch-512 step-4000 checkpoint with resume state (2026-11-17)
ab621ab verified
Raw History Blame Contribute Delete
22.4 kB
"""ACT (Zhao et al. 2023, "Learning Fine-Grained Bimanual Manipulation with Low-Cost Hardware") network core: the
ResNet backbone with frozen BatchNorm and sine position embedding, the DETR encoder-decoder transformer and the CVAE
action-chunk decoder.
Ported unchanged from DextraVerse (`dextraverse/algorithms/il/models/nets/act/`), which vendors the ACT repository's
`detr/models/{backbone,position_encoding,transformer,detr_vae}.py` (Apache-2.0, (c) Facebook, Inc. and its
affiliates; see THIRD_PARTY_NOTICES.md) with the NestedTensor/argparse plumbing removed, explicit `qpos_dim` /
`action_dim` instead of the ALOHA 14-DoF constants, and one backbone shared by every camera (upstream behaviour).
The policy uses `return_intermediate_dec=False`: `Transformer.forward(...)[0]` is the final decoder layer.
"""
from __future__ import annotations
import copy
import math
from typing import Optional
import numpy as np
import torch
import torch.nn.functional as F
import torchvision
from torch import Tensor, nn
from torchvision.models._utils import IntermediateLayerGetter
class PositionEmbeddingSine(nn.Module):
"""Standard DETR sine embedding over a ``(B, C, H, W)`` feature map."""
def __init__(
self, num_pos_feats: int = 64, temperature: int = 10000, normalize: bool = False
) -> None:
super().__init__()
self.num_pos_feats = num_pos_feats
self.temperature = temperature
self.normalize = normalize
self.scale = 2 * math.pi
def forward(self, x: torch.Tensor) -> torch.Tensor:
not_mask = torch.ones((1, x.shape[-2], x.shape[-1]), device=x.device)
y_embed = not_mask.cumsum(1, dtype=torch.float32)
x_embed = not_mask.cumsum(2, dtype=torch.float32)
if self.normalize:
eps = 1e-6
y_embed = y_embed / (y_embed[:, -1:, :] + eps) * self.scale
x_embed = x_embed / (x_embed[:, :, -1:] + eps) * self.scale
dim_t = torch.arange(self.num_pos_feats, dtype=torch.float32, device=x.device)
dim_t = self.temperature ** (2 * (dim_t // 2) / self.num_pos_feats)
pos_x = x_embed[:, :, :, None] / dim_t
pos_y = y_embed[:, :, :, None] / dim_t
pos_x = torch.stack(
(pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4
).flatten(3)
pos_y = torch.stack(
(pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4
).flatten(3)
return torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2)
class FrozenBatchNorm2d(nn.Module):
"""BatchNorm2d with fixed statistics and affine parameters (the DETR/ACT backbone norm)."""
def __init__(self, n: int) -> None:
super().__init__()
self.register_buffer("weight", torch.ones(n))
self.register_buffer("bias", torch.zeros(n))
self.register_buffer("running_mean", torch.zeros(n))
self.register_buffer("running_var", torch.ones(n))
def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, *args):
state_dict.pop(prefix + "num_batches_tracked", None)
super()._load_from_state_dict(state_dict, prefix, local_metadata, strict, *args)
def forward(self, x: torch.Tensor) -> torch.Tensor:
w = self.weight.reshape(1, -1, 1, 1)
b = self.bias.reshape(1, -1, 1, 1)
rv = self.running_var.reshape(1, -1, 1, 1)
rm = self.running_mean.reshape(1, -1, 1, 1)
scale = w * (rv + 1e-5).rsqrt()
return x * scale + (b - rm * scale)
class Backbone(nn.Module):
"""ResNet trunk returning the layer4 feature map."""
def __init__(self, name: str = "resnet18", weights: str | None = None) -> None:
super().__init__()
trunk = getattr(torchvision.models, name)(
replace_stride_with_dilation=[False, False, False],
weights=weights,
norm_layer=FrozenBatchNorm2d,
)
self.body = IntermediateLayerGetter(trunk, return_layers={"layer4": "0"})
self.num_channels = 512 if name in ("resnet18", "resnet34") else 2048
def forward(self, x: torch.Tensor) -> dict[str, torch.Tensor]:
return self.body(x)
class Joiner(nn.Sequential):
"""Backbone + position embedding, returning matched feature/position lists."""
def __init__(self, backbone: Backbone, position_embedding: PositionEmbeddingSine) -> None:
super().__init__(backbone, position_embedding)
self.num_channels = backbone.num_channels
def forward(self, x: torch.Tensor) -> tuple[list[torch.Tensor], list[torch.Tensor]]:
features = list(self[0](x).values())
pos = [self[1](f).to(f.dtype) for f in features]
return features, pos
def build_backbone(
name: str = "resnet18", hidden_dim: int = 512, weights: str | None = None
) -> Joiner:
"""The ACT camera backbone: ResNet + sine embedding sized to the transformer width."""
return Joiner(Backbone(name, weights=weights), PositionEmbeddingSine(hidden_dim // 2))
class Transformer(nn.Module):
def __init__(
self,
d_model=512,
nhead=8,
num_encoder_layers=6,
num_decoder_layers=6,
dim_feedforward=2048,
dropout=0.1,
activation="relu",
normalize_before=False,
return_intermediate_dec=False,
):
super().__init__()
encoder_layer = TransformerEncoderLayer(
d_model, nhead, dim_feedforward, dropout, activation, normalize_before
)
encoder_norm = nn.LayerNorm(d_model) if normalize_before else None
self.encoder = TransformerEncoder(encoder_layer, num_encoder_layers, encoder_norm)
decoder_layer = TransformerDecoderLayer(
d_model, nhead, dim_feedforward, dropout, activation, normalize_before
)
decoder_norm = nn.LayerNorm(d_model)
self.decoder = TransformerDecoder(
decoder_layer,
num_decoder_layers,
decoder_norm,
return_intermediate=return_intermediate_dec,
)
self._reset_parameters()
self.d_model = d_model
self.nhead = nhead
def _reset_parameters(self):
for p in self.parameters():
if p.dim() > 1:
nn.init.xavier_uniform_(p)
def forward(
self,
src,
mask,
query_embed,
pos_embed,
latent_input=None,
proprio_input=None,
additional_pos_embed=None,
):
# TODO flatten only when input has H and W
if len(src.shape) == 4: # has H and W
# flatten NxCxHxW to HWxNxC
bs, c, h, w = src.shape
src = src.flatten(2).permute(2, 0, 1)
pos_embed = pos_embed.flatten(2).permute(2, 0, 1).repeat(1, bs, 1)
query_embed = query_embed.unsqueeze(1).repeat(1, bs, 1)
# mask = mask.flatten(1)
additional_pos_embed = additional_pos_embed.unsqueeze(1).repeat(
1, bs, 1
) # seq, bs, dim
pos_embed = torch.cat([additional_pos_embed, pos_embed], axis=0)
addition_input = torch.stack([latent_input, proprio_input], axis=0)
src = torch.cat([addition_input, src], axis=0)
else:
assert len(src.shape) == 3
# flatten NxHWxC to HWxNxC
bs, hw, c = src.shape
src = src.permute(1, 0, 2)
pos_embed = pos_embed.unsqueeze(1).repeat(1, bs, 1)
query_embed = query_embed.unsqueeze(1).repeat(1, bs, 1)
tgt = torch.zeros_like(query_embed)
memory = self.encoder(src, src_key_padding_mask=mask, pos=pos_embed)
hs = self.decoder(
tgt, memory, memory_key_padding_mask=mask, pos=pos_embed, query_pos=query_embed
)
hs = hs.transpose(1, 2)
return hs
class TransformerEncoder(nn.Module):
def __init__(self, encoder_layer, num_layers, norm=None):
super().__init__()
self.layers = _get_clones(encoder_layer, num_layers)
self.num_layers = num_layers
self.norm = norm
def forward(
self,
src,
mask: Optional[Tensor] = None,
src_key_padding_mask: Optional[Tensor] = None,
pos: Optional[Tensor] = None,
):
output = src
for layer in self.layers:
output = layer(
output, src_mask=mask, src_key_padding_mask=src_key_padding_mask, pos=pos
)
if self.norm is not None:
output = self.norm(output)
return output
class TransformerDecoder(nn.Module):
def __init__(self, decoder_layer, num_layers, norm=None, return_intermediate=False):
super().__init__()
self.layers = _get_clones(decoder_layer, num_layers)
self.num_layers = num_layers
self.norm = norm
self.return_intermediate = return_intermediate
def forward(
self,
tgt,
memory,
tgt_mask: Optional[Tensor] = None,
memory_mask: Optional[Tensor] = None,
tgt_key_padding_mask: Optional[Tensor] = None,
memory_key_padding_mask: Optional[Tensor] = None,
pos: Optional[Tensor] = None,
query_pos: Optional[Tensor] = None,
):
output = tgt
intermediate = []
for layer in self.layers:
output = layer(
output,
memory,
tgt_mask=tgt_mask,
memory_mask=memory_mask,
tgt_key_padding_mask=tgt_key_padding_mask,
memory_key_padding_mask=memory_key_padding_mask,
pos=pos,
query_pos=query_pos,
)
if self.return_intermediate:
intermediate.append(self.norm(output))
if self.norm is not None:
output = self.norm(output)
if self.return_intermediate:
intermediate.pop()
intermediate.append(output)
if self.return_intermediate:
return torch.stack(intermediate)
return output.unsqueeze(0)
class TransformerEncoderLayer(nn.Module):
def __init__(
self,
d_model,
nhead,
dim_feedforward=2048,
dropout=0.1,
activation="relu",
normalize_before=False,
):
super().__init__()
self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)
# Implementation of Feedforward model
self.linear1 = nn.Linear(d_model, dim_feedforward)
self.dropout = nn.Dropout(dropout)
self.linear2 = nn.Linear(dim_feedforward, d_model)
self.norm1 = nn.LayerNorm(d_model)
self.norm2 = nn.LayerNorm(d_model)
self.dropout1 = nn.Dropout(dropout)
self.dropout2 = nn.Dropout(dropout)
self.activation = _get_activation_fn(activation)
self.normalize_before = normalize_before
def with_pos_embed(self, tensor, pos: Optional[Tensor]):
return tensor if pos is None else tensor + pos
def forward_post(
self,
src,
src_mask: Optional[Tensor] = None,
src_key_padding_mask: Optional[Tensor] = None,
pos: Optional[Tensor] = None,
):
q = k = self.with_pos_embed(src, pos)
src2 = self.self_attn(
q, k, value=src, attn_mask=src_mask, key_padding_mask=src_key_padding_mask
)[0]
src = src + self.dropout1(src2)
src = self.norm1(src)
src2 = self.linear2(self.dropout(self.activation(self.linear1(src))))
src = src + self.dropout2(src2)
src = self.norm2(src)
return src
def forward_pre(
self,
src,
src_mask: Optional[Tensor] = None,
src_key_padding_mask: Optional[Tensor] = None,
pos: Optional[Tensor] = None,
):
src2 = self.norm1(src)
q = k = self.with_pos_embed(src2, pos)
src2 = self.self_attn(
q, k, value=src2, attn_mask=src_mask, key_padding_mask=src_key_padding_mask
)[0]
src = src + self.dropout1(src2)
src2 = self.norm2(src)
src2 = self.linear2(self.dropout(self.activation(self.linear1(src2))))
src = src + self.dropout2(src2)
return src
def forward(
self,
src,
src_mask: Optional[Tensor] = None,
src_key_padding_mask: Optional[Tensor] = None,
pos: Optional[Tensor] = None,
):
if self.normalize_before:
return self.forward_pre(src, src_mask, src_key_padding_mask, pos)
return self.forward_post(src, src_mask, src_key_padding_mask, pos)
class TransformerDecoderLayer(nn.Module):
def __init__(
self,
d_model,
nhead,
dim_feedforward=2048,
dropout=0.1,
activation="relu",
normalize_before=False,
):
super().__init__()
self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)
self.multihead_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)
# Implementation of Feedforward model
self.linear1 = nn.Linear(d_model, dim_feedforward)
self.dropout = nn.Dropout(dropout)
self.linear2 = nn.Linear(dim_feedforward, d_model)
self.norm1 = nn.LayerNorm(d_model)
self.norm2 = nn.LayerNorm(d_model)
self.norm3 = nn.LayerNorm(d_model)
self.dropout1 = nn.Dropout(dropout)
self.dropout2 = nn.Dropout(dropout)
self.dropout3 = nn.Dropout(dropout)
self.activation = _get_activation_fn(activation)
self.normalize_before = normalize_before
def with_pos_embed(self, tensor, pos: Optional[Tensor]):
return tensor if pos is None else tensor + pos
def forward_post(
self,
tgt,
memory,
tgt_mask: Optional[Tensor] = None,
memory_mask: Optional[Tensor] = None,
tgt_key_padding_mask: Optional[Tensor] = None,
memory_key_padding_mask: Optional[Tensor] = None,
pos: Optional[Tensor] = None,
query_pos: Optional[Tensor] = None,
):
q = k = self.with_pos_embed(tgt, query_pos)
tgt2 = self.self_attn(
q, k, value=tgt, attn_mask=tgt_mask, key_padding_mask=tgt_key_padding_mask
)[0]
tgt = tgt + self.dropout1(tgt2)
tgt = self.norm1(tgt)
tgt2 = self.multihead_attn(
query=self.with_pos_embed(tgt, query_pos),
key=self.with_pos_embed(memory, pos),
value=memory,
attn_mask=memory_mask,
key_padding_mask=memory_key_padding_mask,
)[0]
tgt = tgt + self.dropout2(tgt2)
tgt = self.norm2(tgt)
tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt))))
tgt = tgt + self.dropout3(tgt2)
tgt = self.norm3(tgt)
return tgt
def forward_pre(
self,
tgt,
memory,
tgt_mask: Optional[Tensor] = None,
memory_mask: Optional[Tensor] = None,
tgt_key_padding_mask: Optional[Tensor] = None,
memory_key_padding_mask: Optional[Tensor] = None,
pos: Optional[Tensor] = None,
query_pos: Optional[Tensor] = None,
):
tgt2 = self.norm1(tgt)
q = k = self.with_pos_embed(tgt2, query_pos)
tgt2 = self.self_attn(
q, k, value=tgt2, attn_mask=tgt_mask, key_padding_mask=tgt_key_padding_mask
)[0]
tgt = tgt + self.dropout1(tgt2)
tgt2 = self.norm2(tgt)
tgt2 = self.multihead_attn(
query=self.with_pos_embed(tgt2, query_pos),
key=self.with_pos_embed(memory, pos),
value=memory,
attn_mask=memory_mask,
key_padding_mask=memory_key_padding_mask,
)[0]
tgt = tgt + self.dropout2(tgt2)
tgt2 = self.norm3(tgt)
tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt2))))
tgt = tgt + self.dropout3(tgt2)
return tgt
def forward(
self,
tgt,
memory,
tgt_mask: Optional[Tensor] = None,
memory_mask: Optional[Tensor] = None,
tgt_key_padding_mask: Optional[Tensor] = None,
memory_key_padding_mask: Optional[Tensor] = None,
pos: Optional[Tensor] = None,
query_pos: Optional[Tensor] = None,
):
if self.normalize_before:
return self.forward_pre(
tgt,
memory,
tgt_mask,
memory_mask,
tgt_key_padding_mask,
memory_key_padding_mask,
pos,
query_pos,
)
return self.forward_post(
tgt,
memory,
tgt_mask,
memory_mask,
tgt_key_padding_mask,
memory_key_padding_mask,
pos,
query_pos,
)
def _get_clones(module, N):
return nn.ModuleList([copy.deepcopy(module) for i in range(N)])
def _get_activation_fn(activation):
"""Return an activation function given a string."""
if activation == "relu":
return F.relu
if activation == "gelu":
return F.gelu
raise RuntimeError(f"activation should be relu/gelu, not {activation}.")
def reparametrize(mu: torch.Tensor, logvar: torch.Tensor) -> torch.Tensor:
std = logvar.div(2).exp()
return mu + std * torch.randn_like(std)
def get_sinusoid_encoding_table(n_position: int, d_hid: int) -> torch.Tensor:
def angle_vec(position: int) -> list[float]:
return [position / np.power(10000, 2 * (j // 2) / d_hid) for j in range(d_hid)]
table = np.array([angle_vec(i) for i in range(n_position)])
table[:, 0::2] = np.sin(table[:, 0::2])
table[:, 1::2] = np.cos(table[:, 1::2])
return torch.FloatTensor(table).unsqueeze(0)
def build_style_encoder(
hidden_dim: int, nheads: int, dim_feedforward: int, enc_layers: int, dropout: float
) -> TransformerEncoder:
"""The CVAE style encoder (a plain transformer encoder over [CLS, qpos, action seq])."""
layer = TransformerEncoderLayer(hidden_dim, nheads, dim_feedforward, dropout, "relu", False)
return TransformerEncoder(layer, enc_layers, norm=None)
class DETRVAE(nn.Module):
"""CVAE decoder: (qpos, images[, action seq]) → ``num_queries`` future actions."""
def __init__(
self,
backbone: Joiner,
transformer: Transformer,
encoder: TransformerEncoder,
*,
qpos_dim: int,
action_dim: int,
num_queries: int,
latent_dim: int = 32,
) -> None:
super().__init__()
self.num_queries = num_queries
self.transformer = transformer
self.encoder = encoder
hidden_dim = transformer.d_model
self.action_head = nn.Linear(hidden_dim, action_dim)
self.query_embed = nn.Embedding(num_queries, hidden_dim)
self.backbone = backbone # ONE backbone shared by every camera (upstream behavior)
self.input_proj = nn.Conv2d(backbone.num_channels, hidden_dim, kernel_size=1)
self.input_proj_robot_state = nn.Linear(qpos_dim, hidden_dim)
# CVAE style encoder inputs
self.latent_dim = latent_dim
self.cls_embed = nn.Embedding(1, hidden_dim)
self.encoder_action_proj = nn.Linear(action_dim, hidden_dim)
self.encoder_joint_proj = nn.Linear(qpos_dim, hidden_dim)
self.latent_proj = nn.Linear(hidden_dim, latent_dim * 2)
self.register_buffer(
"pos_table", get_sinusoid_encoding_table(1 + 1 + num_queries, hidden_dim)
)
# decoder side
self.latent_out_proj = nn.Linear(latent_dim, hidden_dim)
self.additional_pos_embed = nn.Embedding(2, hidden_dim) # latent + proprio slots
def forward(
self,
qpos: torch.Tensor,
images: torch.Tensor,
actions: torch.Tensor | None = None,
is_pad: torch.Tensor | None = None,
) -> tuple[torch.Tensor, tuple[torch.Tensor | None, torch.Tensor | None]]:
"""``qpos (B, Dq)``, ``images (B, ncam, C, H, W)``; train passes ``actions (B, T, Da)``."""
is_training = actions is not None
bs = qpos.shape[0]
if is_training:
action_embed = self.encoder_action_proj(actions)
qpos_embed = self.encoder_joint_proj(qpos).unsqueeze(1)
cls_embed = self.cls_embed.weight.unsqueeze(0).repeat(bs, 1, 1)
encoder_input = torch.cat([cls_embed, qpos_embed, action_embed], dim=1)
encoder_input = encoder_input.permute(1, 0, 2)
if is_pad is None:
is_pad = torch.zeros(bs, actions.shape[1], dtype=torch.bool, device=qpos.device)
cls_joint_is_pad = torch.full((bs, 2), False, device=qpos.device)
key_pad = torch.cat([cls_joint_is_pad, is_pad], dim=1)
pos_embed = self.pos_table.clone().detach().permute(1, 0, 2)
encoder_output = self.encoder(
encoder_input, pos=pos_embed, src_key_padding_mask=key_pad
)[0]
latent_info = self.latent_proj(encoder_output)
mu = latent_info[:, : self.latent_dim]
logvar = latent_info[:, self.latent_dim :]
latent_input = self.latent_out_proj(reparametrize(mu, logvar))
else:
mu = logvar = None
latent_sample = torch.zeros(bs, self.latent_dim, device=qpos.device)
latent_input = self.latent_out_proj(latent_sample)
all_cam_features, all_cam_pos = [], []
for cam_id in range(images.shape[1]):
features, pos = self.backbone(images[:, cam_id])
all_cam_features.append(self.input_proj(features[0]))
all_cam_pos.append(pos[0])
proprio_input = self.input_proj_robot_state(qpos)
src = torch.cat(all_cam_features, dim=3) # cameras fold into the width axis
pos = torch.cat(all_cam_pos, dim=3)
hs = self.transformer(
src,
None,
self.query_embed.weight,
pos,
latent_input,
proprio_input,
self.additional_pos_embed.weight,
)[0]
return self.action_head(hs), (mu, logvar)
def kl_divergence(mu: torch.Tensor, logvar: torch.Tensor) -> torch.Tensor:
"""Total KL(q(z|x) || N(0, I)), summed over latent dims, meaned over the batch."""
klds = -0.5 * (1 + logvar - mu.pow(2) - logvar.exp())
return klds.sum(1).mean()