File size: 9,986 Bytes
ab621ab
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
"""Networks of the FM-DiT and DP (DDPM-DiT) baselines: the AdaLN DiT action backbone and the camera + state encoder.

Ported from DextraVerse's imitation-learning core (`dextraverse/algorithms/il/models/nets/dit.py`, `layers.py`,
`positional_embedding.py`, `vision/backbones.py`, `vision/encoder.py`), unchanged in architecture and initialization.
Left out: the ROVE improvement embedding, the spatial-softmax / R3M / tactile options and the inpainting path (the
baselines always condition through the global observation feature). Added: the ResNet's normalization choice
(`get_resnet`): DextraVerse's GroupNorm for a backbone trained from scratch, or BatchNorm frozen at the pretrained
statistics (the DETR / ACT practice) for an ImageNet-initialized backbone that is fine-tuned with the policy.
"""
from __future__ import annotations

import math

import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision

IMAGENET_MEAN = (0.485, 0.456, 0.406)
IMAGENET_STD = (0.229, 0.224, 0.225)


class SinusoidalPosEmb(nn.Module):
    def __init__(self, dim):
        super().__init__()
        self.dim = dim

    def forward(self, x):
        half_dim = self.dim // 2
        emb = math.log(10000) / (half_dim - 1)
        emb = torch.exp(torch.arange(half_dim, device=x.device) * -emb)
        emb = x[:, None] * emb[None, :]
        return torch.cat((emb.sin(), emb.cos()), dim=-1)


class RotaryPosEmb(nn.Module):
    """Rotary position embedding over each attention head's (S, head_dim) slice."""

    def __init__(self, dim):
        super().__init__()
        self.dim = dim
        self.register_buffer('inv_freq', 1.0 / (10000 ** (torch.arange(0, dim, 2).float() / dim)))

    def forward(self, x):  # x: (B, heads, S, head_dim)
        t = torch.arange(x.shape[-2], device=x.device, dtype=self.inv_freq.dtype)
        freqs = torch.outer(t, self.inv_freq.to(x.device))
        emb = torch.cat([freqs, freqs], dim=-1)
        cos, sin = emb.cos()[None, None], emb.sin()[None, None]
        x1, x2 = x[..., : x.shape[-1] // 2], x[..., x.shape[-1] // 2:]
        return x * cos + torch.cat([-x2, x1], dim=-1) * sin


class Mlp(nn.Module):
    """The ViT MLP (timm.layers.Mlp) as DextraVerse's nets/layers.py."""

    def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.0):
        super().__init__()
        out_features = out_features or in_features
        hidden_features = hidden_features or in_features
        self.fc1 = nn.Linear(in_features, hidden_features)
        self.act = act_layer()
        self.drop1 = nn.Dropout(drop)
        self.norm = nn.Identity()
        self.fc2 = nn.Linear(hidden_features, out_features)
        self.drop2 = nn.Dropout(drop)

    def forward(self, x):
        return self.drop2(self.fc2(self.norm(self.drop1(self.act(self.fc1(x))))))


class Attention(nn.Module):
    def __init__(self, dim, num_heads=8, qkv_bias=True, attn_drop=0.0, proj_drop=0.0, qk_norm=True):
        super().__init__()
        assert dim % num_heads == 0, 'dim should be divisible by num_heads'
        self.num_heads = num_heads
        self.head_dim = dim // num_heads
        self.attn_drop = attn_drop
        self.proj_drop = nn.Dropout(proj_drop)
        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
        self.q_norm = nn.LayerNorm(self.head_dim) if qk_norm else nn.Identity()
        self.k_norm = nn.LayerNorm(self.head_dim) if qk_norm else nn.Identity()
        self.proj = nn.Linear(dim, dim)
        self.rope = RotaryPosEmb(self.head_dim)

    def forward(self, x):
        B, S, C = x.shape
        qkv = self.qkv(x).reshape(B, S, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4)
        q, k, v = qkv.unbind(0)
        q, k = self.rope(self.q_norm(q)), self.rope(self.k_norm(k))
        x = F.scaled_dot_product_attention(q, k, v, dropout_p=self.attn_drop if self.training else 0.0)
        return self.proj_drop(self.proj(x.transpose(1, 2).reshape(B, S, C)))


class AdaLNBlock(nn.Module):
    """Adaptive-LayerNorm transformer block as in DiT (Peebles et al. 2022)."""

    def __init__(self, dim, num_heads, mlp_ratio=4.0, dropout=0.0):
        super().__init__()
        self.norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
        self.attn = Attention(dim, num_heads=num_heads, attn_drop=dropout, proj_drop=dropout)
        self.norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
        self.mlp = Mlp(in_features=dim, hidden_features=int(dim * mlp_ratio),
                       act_layer=lambda: nn.GELU(approximate='tanh'), drop=0)
        self.ada_ln = nn.Sequential(nn.SiLU(), nn.Linear(dim, 6 * dim))
        self.dim = dim
        nn.init.constant_(self.ada_ln[-1].weight, 0)
        nn.init.constant_(self.ada_ln[-1].bias, 0)

    def forward(self, x, t, c):
        B = x.shape[0]
        gamma1, gamma2, scale1, scale2, shift1, shift2 = self.ada_ln(F.silu(t + c)).view(B, 6, 1, self.dim).unbind(1)
        x = x + self.attn(self.norm1(x).mul(scale1.add(1)).add_(shift1)).mul_(gamma1)
        return x + self.mlp(self.norm2(x).mul(scale2.add(1)).add_(shift2)).mul_(gamma2)


class FlowTransformer(nn.Module):
    """DiT action backbone: AdaLN blocks over the H action tokens, conditioned on time_embed(t) + cond_embed(obs).

    As DextraVerse, the generic Xavier initialization runs after the blocks' zero init of their AdaLN projection, so
    it is not the full AdaLN-Zero recipe (kept: it is the reference's training behaviour)."""

    def __init__(self, input_dim, condition_dim, hidden_dim, output_dim, num_layers, num_heads, mlp_ratio=4.0,
                 dropout=0.1, time_embed_dim=256):
        super().__init__()
        self.input_proj = nn.Linear(input_dim, hidden_dim)
        self.time_embed = nn.Sequential(SinusoidalPosEmb(time_embed_dim), nn.Linear(time_embed_dim, time_embed_dim * 4),
                                        nn.Mish(), nn.Linear(time_embed_dim * 4, hidden_dim))
        self.cond_embed = nn.Linear(condition_dim, hidden_dim)
        self.transformer_blocks = nn.ModuleList(
            [AdaLNBlock(hidden_dim, num_heads, mlp_ratio=mlp_ratio, dropout=dropout) for _ in range(num_layers)])
        self.norm = nn.LayerNorm(hidden_dim)
        self.out_proj = nn.Linear(hidden_dim, output_dim)

        def basic_init(module):
            if isinstance(module, nn.Linear):
                nn.init.xavier_uniform_(module.weight)
                if module.bias is not None:
                    nn.init.constant_(module.bias, 0)

        self.apply(basic_init)
        nn.init.normal_(self.time_embed[1].weight, std=0.02)
        nn.init.normal_(self.time_embed[3].weight, std=0.02)

    def forward(self, x, t, global_cond):
        if not torch.is_tensor(t):
            t = torch.tensor(t, device=x.device)
        if t.ndim == 0:
            t = t.expand(x.shape[0])
        x = self.input_proj(x)
        t = self.time_embed(t)
        c = self.cond_embed(global_cond)
        for block in self.transformer_blocks:
            x = block(x, t, c)
        return self.out_proj(self.norm(x))


def replace_batchnorm_with_groupnorm(module: nn.Module) -> nn.Module:
    """DextraVerse's use_group_norm: every BatchNorm2d becomes GroupNorm(num_features // 16 groups)."""
    for name, child in module.named_children():
        if isinstance(child, nn.BatchNorm2d):
            setattr(module, name, nn.GroupNorm(num_groups=child.num_features // 16, num_channels=child.num_features))
        else:
            replace_batchnorm_with_groupnorm(child)
    return module


NORMS = ('group', 'frozen_bn')


def get_resnet(name='resnet18', weights=None, norm='group') -> nn.Module:
    """A torchvision ResNet with its classifier removed: (N, 3, H, W) -> (N, 512) globally pooled.

    weights: torchvision weights name (e.g. 'IMAGENET1K_V1') to initialize from, or None (random initialization;
    a checkpoint load supplies every weight). norm: 'group' replaces every BatchNorm with GroupNorm (DextraVerse's
    use_group_norm, for training from scratch: pretrained BatchNorm statistics and affine are dropped); 'frozen_bn'
    keeps BatchNorm frozen at its (pretrained) running statistics and affine, the convolutions fine-tune."""
    if norm == 'frozen_bn':
        from .act_nets import FrozenBatchNorm2d
        resnet = getattr(torchvision.models, name)(weights=weights, norm_layer=FrozenBatchNorm2d)
    elif norm == 'group':
        resnet = replace_batchnorm_with_groupnorm(getattr(torchvision.models, name)(weights=weights))
    else:
        raise ValueError(f'norm must be one of {NORMS}, got {norm!r}')
    resnet.fc = nn.Identity()
    return resnet


class ImageStateEncoder(nn.Module):
    """DextraVerse MultiImageObsEncoder as configured for FM-DiT / DDPM-DiT: one ResNet shared by every camera
    (share_rgb_model; `get_resnet` weights and norm), ImageNet normalization of the [0, 1] images, the pooled features
    of each camera in the contract's camera order, then the normalized state.
    Output (B, n_cameras * 512 + state_dim)."""

    def __init__(self, n_cameras, state_dim, backbone='resnet18', weights=None, norm='group'):
        super().__init__()
        self.rgb = get_resnet(backbone, weights=weights, norm=norm)
        self.n_cameras, self.state_dim = int(n_cameras), int(state_dim)
        self.register_buffer('mean', torch.tensor(IMAGENET_MEAN).view(1, 3, 1, 1), persistent=False)
        self.register_buffer('std', torch.tensor(IMAGENET_STD).view(1, 3, 1, 1), persistent=False)
        self.feature_dim = self.n_cameras * (512 if backbone in ('resnet18', 'resnet34') else 2048) + self.state_dim

    def forward(self, images, state):
        """images (B, n_cameras, 3, H, W) in [0, 1]; state (B, state_dim) normalized."""
        B = images.shape[0]
        x = (images.flatten(0, 1) - self.mean) / self.std
        features = self.rgb(x).reshape(B, -1)  # camera features side by side, in camera order
        return torch.cat([features, state], dim=-1)