""" References: - DiT: https://github.com/facebookresearch/DiT/blob/main/models.py - Diffusion Forcing: https://github.com/buoyancy99/diffusion-forcing/blob/main/algorithms/diffusion_forcing/models/unet3d.py - Latte: https://github.com/Vchitect/Latte/blob/main/models/latte.py """ from typing import Optional, Literal import torch from torch import nn from torch.nn import functional as F from .rotary_embedding_torch import RotaryEmbedding from einops import rearrange from .attention import SpatialAxialAttention, TemporalAxialAttention from timm.models.vision_transformer import Mlp from timm.layers.helpers import to_2tuple import math from collections import namedtuple from typing import Optional, Callable from .cameractrl_module import SimpleCameraPoseEncoder def modulate(x, shift, scale): fixed_dims = [1] * len(shift.shape[1:]) shift = shift.repeat(x.shape[0] // shift.shape[0], *fixed_dims) scale = scale.repeat(x.shape[0] // scale.shape[0], *fixed_dims) while shift.dim() < x.dim(): shift = shift.unsqueeze(-2) scale = scale.unsqueeze(-2) return x * (1 + scale) + shift def gate(x, g): fixed_dims = [1] * len(g.shape[1:]) g = g.repeat(x.shape[0] // g.shape[0], *fixed_dims) while g.dim() < x.dim(): g = g.unsqueeze(-2) return g * x class PatchEmbed(nn.Module): """2D Image to Patch Embedding""" def __init__( self, img_height=256, img_width=256, patch_size=16, in_chans=3, embed_dim=768, norm_layer=None, flatten=True, ): super().__init__() img_size = (img_height, img_width) patch_size = to_2tuple(patch_size) self.img_size = img_size self.patch_size = patch_size self.grid_size = (img_size[0] // patch_size[0], img_size[1] // patch_size[1]) self.num_patches = self.grid_size[0] * self.grid_size[1] self.flatten = flatten self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() def forward(self, x, random_sample=False): B, C, H, W = x.shape assert random_sample or (H == self.img_size[0] and W == self.img_size[1]), f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})." x = self.proj(x) if self.flatten: x = rearrange(x, "B C H W -> B (H W) C") else: x = rearrange(x, "B C H W -> B H W C") x = self.norm(x) return x class TimestepEmbedder(nn.Module): """ Embeds scalar timesteps into vector representations. """ def __init__(self, hidden_size, frequency_embedding_size=256, freq_type='time_step'): super().__init__() self.mlp = nn.Sequential( nn.Linear(frequency_embedding_size, hidden_size, bias=True), # hidden_size is diffusion model hidden size nn.SiLU(), nn.Linear(hidden_size, hidden_size, bias=True), ) self.frequency_embedding_size = frequency_embedding_size self.freq_type = freq_type @staticmethod def timestep_embedding(t, dim, max_period=10000, freq_type='time_step'): """ Create sinusoidal timestep embeddings. :param t: a 1-D Tensor of N indices, one per batch element. These may be fractional. :param dim: the dimension of the output. :param max_period: controls the minimum frequency of the embeddings. :return: an (N, D) Tensor of positional embeddings. """ # https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.py half = dim // 2 if freq_type == 'time_step': freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half).to(device=t.device) elif freq_type == 'spatial': # ~(-5 5) freqs = torch.linspace(1.0, half, half).to(device=t.device) * torch.pi elif freq_type == 'angle': # 0-360 freqs = torch.linspace(1.0, half, half).to(device=t.device) * torch.pi / 180 args = t[:, None].float() * freqs[None] embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) if dim % 2: embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) return embedding def forward(self, t): t_freq = self.timestep_embedding(t, self.frequency_embedding_size, freq_type=self.freq_type) t_emb = self.mlp(t_freq) return t_emb class FinalLayer(nn.Module): """ The final layer of DiT. """ def __init__(self, hidden_size, patch_size, out_channels): super().__init__() self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) self.linear = nn.Linear(hidden_size, patch_size * patch_size * out_channels, bias=True) self.adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=True)) def forward(self, x, c): shift, scale = self.adaLN_modulation(c).chunk(2, dim=-1) x = modulate(self.norm_final(x), shift, scale) x = self.linear(x) return x class FrameMemoryReferenceAttention(nn.Module): def __init__(self, hidden_size, num_heads, key_only_geometry=True): super().__init__() self.hidden_size = hidden_size self.num_heads = num_heads self.head_dim = hidden_size // num_heads self.key_only_geometry = key_only_geometry self.to_q = nn.Linear(hidden_size, hidden_size, bias=False) self.to_k = nn.Linear(hidden_size, hidden_size, bias=False) self.to_v = nn.Linear(hidden_size, hidden_size, bias=False) self.query_pose_proj = nn.Linear(6, hidden_size, bias=True) self.key_pose_proj = nn.Linear(6, hidden_size, bias=True) self.out_proj = nn.Linear(hidden_size, hidden_size, bias=True) def _split_heads(self, x): return rearrange(x, "b n (h d) -> b h n d", h=self.num_heads) def forward(self, target_hidden, memory_hidden, memory_mask=None, geometry_cache=None): B, T_target, H, W, D = target_hidden.shape if memory_hidden is None or int(memory_hidden.shape[1]) == 0: return target_hidden.new_zeros(target_hidden.shape) T_memory = memory_hidden.shape[1] P = H * W if geometry_cache is not None: query_rays = None if self.key_only_geometry else geometry_cache.get("query_rays") if query_rays is not None: query_rays = query_rays.to(device=target_hidden.device, dtype=target_hidden.dtype) relative_rays = geometry_cache.get("relative_rays") if relative_rays is not None: relative_rays = relative_rays.to(device=target_hidden.device, dtype=target_hidden.dtype) target_time = None if self.key_only_geometry else geometry_cache.get("target_time") relative_time = geometry_cache.get("relative_time") else: query_rays = None relative_rays = None target_time = None relative_time = None q_input = target_hidden if query_rays is not None: # Values stay visual-only; geometry and time only steer Q/K matching. q_input = q_input + self.query_pose_proj(query_rays) if target_time is not None: q_input = q_input + target_time.to(device=target_hidden.device, dtype=target_hidden.dtype)[:, :, None, None] q = rearrange(q_input, "b t h w d -> (b t) (h w) d") q = self._split_heads(self.to_q(q)) memory_tokens = rearrange(memory_hidden, "b m h w d -> b m (h w) d") memory_flat = rearrange(memory_tokens, "b m p d -> b (m p) d") key_cond = None if relative_rays is not None: key_cond = self.key_pose_proj( rearrange(relative_rays, "b t m h w r -> b t m (h w) r") ) if relative_time is not None: relative_time = relative_time.to(device=target_hidden.device, dtype=target_hidden.dtype) relative_time = relative_time[:, :, :, None] key_cond = relative_time if key_cond is None else key_cond + relative_time if key_cond is None: k = self._split_heads(self.to_k(memory_flat)) else: k = self.to_k(memory_tokens[:, None] + key_cond) k = self._split_heads(rearrange(k, "b t m p d -> (b t) (m p) d")) v = self._split_heads(self.to_v(memory_flat)) if memory_mask is None: frame_valid = torch.ones((B, T_memory), device=target_hidden.device, dtype=torch.bool) else: frame_valid = memory_mask.to(device=target_hidden.device, dtype=torch.bool) key_valid = frame_valid[:, :, None].expand(B, T_memory, P).reshape(B, T_memory * P) active = key_valid.any(dim=1) key_valid = torch.cat([key_valid, ~active[:, None]], dim=1) row_batch = torch.arange(B, device=target_hidden.device).repeat_interleave(T_target) if key_cond is None: k = k.index_select(0, row_batch) # A zero dummy key keeps all-padded samples finite; the projected delta # is masked back to zero below so padded memory cannot contribute. dummy_k = k.new_zeros((k.shape[0], self.num_heads, 1, self.head_dim)) k = torch.cat([k, dummy_k], dim=2) dummy_v = v.new_zeros((B, self.num_heads, 1, self.head_dim)) v = torch.cat([v, dummy_v], dim=2).index_select(0, row_batch) attn_bias = torch.zeros((B * T_target, 1, 1, key_valid.shape[1]), device=q.device, dtype=q.dtype) attn_bias = attn_bias.masked_fill(~key_valid[row_batch][:, None, None], float("-inf")) x = F.scaled_dot_product_attention( query=q.contiguous(), key=k.contiguous(), value=v.contiguous(), attn_mask=attn_bias, ) x = rearrange(x, "n h p d -> n p (h d)") x = self.out_proj(x).to(dtype=target_hidden.dtype) x = rearrange(x, "(b t) (h w) d -> b t h w d", b=B, t=T_target, h=H, w=W) return x * active[:, None, None, None, None].to(dtype=x.dtype) class SpatioTemporalDiTBlock(nn.Module): def __init__( self, hidden_size, num_heads, reference_length, mlp_ratio=4.0, is_causal=True, spatial_rotary_emb: Optional[RotaryEmbedding] = None, temporal_rotary_emb: Optional[RotaryEmbedding] = None, reference_rotary_emb=None, use_plucker=False, relative_embedding=False, state_embed_only_on_qk=False, use_memory_attention=False, key_only_geometry=True, ref_mode='sequential' ): super().__init__() self.is_causal = is_causal mlp_hidden_dim = int(hidden_size * mlp_ratio) approx_gelu = lambda: nn.GELU(approximate="tanh") self.s_norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) self.s_attn = SpatialAxialAttention( hidden_size, heads=num_heads, dim_head=hidden_size // num_heads, rotary_emb=spatial_rotary_emb ) self.s_norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) self.s_mlp = Mlp( in_features=hidden_size, hidden_features=mlp_hidden_dim, act_layer=approx_gelu, drop=0, ) self.s_adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 6 * hidden_size, bias=True)) self.t_norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) self.t_attn = TemporalAxialAttention( hidden_size, heads=num_heads, dim_head=hidden_size // num_heads, is_causal=is_causal, rotary_emb=temporal_rotary_emb, reference_length=reference_length ) self.t_norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) self.t_mlp = Mlp( in_features=hidden_size, hidden_features=mlp_hidden_dim, act_layer=approx_gelu, drop=0, ) self.t_adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 6 * hidden_size, bias=True)) self.use_memory_attention = use_memory_attention if self.use_memory_attention: self.r_norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) self.r_norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) self.r_mlp = Mlp( in_features=hidden_size, hidden_features=mlp_hidden_dim, act_layer=approx_gelu, drop=0, ) self.r_adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 6 * hidden_size, bias=True)) self.r_attn_anchor = FrameMemoryReferenceAttention(hidden_size, num_heads, key_only_geometry=key_only_geometry) self.r_attn_dynamic = FrameMemoryReferenceAttention(hidden_size, num_heads, key_only_geometry=key_only_geometry) self.r_attn_revisit = FrameMemoryReferenceAttention(hidden_size, num_heads, key_only_geometry=key_only_geometry) self.reference_length = reference_length self.relative_embedding = relative_embedding self.state_embed_only_on_qk = state_embed_only_on_qk self.ref_mode = ref_mode if self.ref_mode == 'parallel': self.parallel_map = nn.Linear(hidden_size, hidden_size) def _split_frame_memory(self, x, frame_memory_segments): target = int(frame_memory_segments.get("target", 0)) anchor = int(frame_memory_segments.get("anchor", 0)) dynamic = int(frame_memory_segments.get("dynamic", 0)) revisit = int(frame_memory_segments.get("revisit", 0)) if min(target, anchor, dynamic, revisit) < 0: raise ValueError( "frame_memory_segments lengths must be nonnegative; " f"got target={target}, anchor={anchor}, dynamic={dynamic}, revisit={revisit}" ) total = target + anchor + dynamic + revisit if total != int(x.shape[1]): raise ValueError( f"frame_memory_segments lengths sum to {total}, expected x.shape[1]={int(x.shape[1])}" ) a0, a1 = target, target + anchor d0, d1 = a1, a1 + dynamic r0, r1 = d1, d1 + revisit return x[:, :target], x[:, a0:a1], x[:, d0:d1], x[:, r0:r1] def _frame_memory_stream_mask(self, frame_memory_masks, stream_name, stream_hidden): self._check_frame_memory_stream_mask_shape(frame_memory_masks, stream_name, stream_hidden) if stream_hidden is None or int(stream_hidden.shape[1]) == 0: return None if frame_memory_masks is None or frame_memory_masks.get(stream_name) is None: return None return frame_memory_masks[stream_name].to(device=stream_hidden.device, dtype=torch.bool) def _check_frame_memory_stream_mask_shape(self, frame_memory_masks, stream_name, stream_hidden): if stream_hidden is None: return if frame_memory_masks is None or frame_memory_masks.get(stream_name) is None: return stream_mask = frame_memory_masks[stream_name] expected_shape = (int(stream_hidden.shape[0]), int(stream_hidden.shape[1])) if tuple(stream_mask.shape) != expected_shape: raise ValueError( f"frame_memory_masks[{stream_name!r}] shape {tuple(stream_mask.shape)} " f"must match {expected_shape}" ) def _apply_frame_memory_reference_attention(self, x, c, frame_memory_segments, frame_memory_masks, frame_memory_geometry=None): x_target, x_anchor, x_dynamic, x_revisit = self._split_frame_memory(x, frame_memory_segments) if int(x_target.shape[1]) == 0: return x r_shift_msa, r_scale_msa, r_gate_msa, r_shift_mlp, r_scale_mlp, r_gate_mlp = self.r_adaLN_modulation(c).chunk(6, dim=-1) attn_input = modulate(self.r_norm1(x), r_shift_msa, r_scale_msa) attn_target, attn_anchor, attn_dynamic, attn_revisit = self._split_frame_memory(attn_input, frame_memory_segments) deltas = [] active_stream_count = x_target.new_zeros((x_target.shape[0],)) for stream_name, r_attn, stream_hidden, stream_attn in ( ("anchor", self.r_attn_anchor, x_anchor, attn_anchor), ("dynamic", self.r_attn_dynamic, x_dynamic, attn_dynamic), ("revisit", self.r_attn_revisit, x_revisit, attn_revisit), ): if int(stream_hidden.shape[1]) == 0: continue stream_mask = self._frame_memory_stream_mask(frame_memory_masks, stream_name, stream_hidden) stream_geometry = None if frame_memory_geometry is None else frame_memory_geometry.get(stream_name) deltas.append(r_attn(attn_target, stream_attn, stream_mask, geometry_cache=stream_geometry)) if stream_mask is None: active_stream_count = active_stream_count + 1 else: active_stream_count = active_stream_count + stream_mask.any(dim=1).to(dtype=active_stream_count.dtype) if deltas: # Average over streams that have at least one valid memory frame per # sample; samples with no active streams receive a zero reference delta. r_delta = sum(deltas) active = active_stream_count > 0 r_delta = r_delta / active_stream_count.clamp_min(1)[:, None, None, None, None] r_delta = r_delta * active[:, None, None, None, None].to(dtype=r_delta.dtype) x_target = x_target + gate(r_delta, r_gate_msa[:, :x_target.shape[1]]) x = torch.cat([x_target, x_anchor, x_dynamic, x_revisit], dim=1) x = x + gate(self.r_mlp(modulate(self.r_norm2(x), r_shift_mlp, r_scale_mlp)), r_gate_mlp) return x def forward(self, x, c, current_frame=None, timestep=None, is_last_block=False, pose_cond=None, mode="training", c_action_cond=None, reference_length=None, frame_memory_segments=None, frame_memory_masks=None, frame_memory_pose=None, image_hw=None, frame_memory_geometry=None): B, T, H, W, D = x.shape if frame_memory_segments is not None: for stream_name, stream_hidden in zip( ("target", "anchor", "dynamic", "revisit"), self._split_frame_memory(x, frame_memory_segments), ): self._check_frame_memory_stream_mask_shape(frame_memory_masks, stream_name, stream_hidden) # spatial block s_shift_msa, s_scale_msa, s_gate_msa, s_shift_mlp, s_scale_mlp, s_gate_mlp = self.s_adaLN_modulation(c).chunk(6, dim=-1) x = x + gate(self.s_attn(modulate(self.s_norm1(x), s_shift_msa, s_scale_msa)), s_gate_msa) x = x + gate(self.s_mlp(modulate(self.s_norm2(x), s_shift_mlp, s_scale_mlp)), s_gate_mlp) # temporal block if c_action_cond is not None: t_shift_msa, t_scale_msa, t_gate_msa, t_shift_mlp, t_scale_mlp, t_gate_mlp = self.t_adaLN_modulation(c_action_cond).chunk(6, dim=-1) else: t_shift_msa, t_scale_msa, t_gate_msa, t_shift_mlp, t_scale_mlp, t_gate_mlp = self.t_adaLN_modulation(c).chunk(6, dim=-1) x_t = x + gate(self.t_attn( modulate(self.t_norm1(x), t_shift_msa, t_scale_msa), frame_memory_segments=frame_memory_segments, frame_memory_masks=frame_memory_masks, ), t_gate_msa) x_t = x_t + gate(self.t_mlp(modulate(self.t_norm2(x_t), t_shift_mlp, t_scale_mlp)), t_gate_mlp) if self.ref_mode == 'sequential': x = x_t # memory block if self.use_memory_attention and frame_memory_segments is not None: x = self._apply_frame_memory_reference_attention( x, c, frame_memory_segments, frame_memory_masks, frame_memory_geometry, ) if self.ref_mode == 'parallel': x = x_t + self.parallel_map(x) return x class DiT(nn.Module): """ Diffusion model with a Transformer backbone. """ def __init__( self, input_h=18, input_w=32, patch_size=2, in_channels=16, hidden_size=1024, depth=12, num_heads=16, mlp_ratio=4.0, action_cond_dim=25, pose_cond_dim=4, max_frames=32, reference_length=8, use_plucker=False, relative_embedding=False, state_embed_only_on_qk=False, use_memory_attention=False, add_timestamp_embedding=False, ref_mode='sequential', focal_length=0.35, memory_attention_key_only_geometry=True, ): super().__init__() self.in_channels = in_channels self.out_channels = in_channels self.patch_size = patch_size self.num_heads = num_heads self.max_frames = max_frames self.focal_length = focal_length self.x_embedder = PatchEmbed(input_h, input_w, patch_size, in_channels, hidden_size, flatten=False) self.t_embedder = TimestepEmbedder(hidden_size) self.add_timestamp_embedding = add_timestamp_embedding if self.add_timestamp_embedding: self.timestamp_embedding = TimestepEmbedder(hidden_size) frame_h, frame_w = self.x_embedder.grid_size self.spatial_rotary_emb = RotaryEmbedding(dim=hidden_size // num_heads // 2, freqs_for="pixel", max_freq=256) self.temporal_rotary_emb = RotaryEmbedding(dim=hidden_size // num_heads) # self.reference_rotary_emb = RotaryEmbedding(dim=hidden_size // num_heads // 2, freqs_for="pixel", max_freq=256) self.reference_rotary_emb = None self.external_cond = nn.Linear(action_cond_dim, hidden_size) if action_cond_dim > 0 else nn.Identity() # self.pose_cond = nn.Linear(pose_cond_dim, hidden_size) if pose_cond_dim > 0 else nn.Identity() self.use_plucker = use_plucker if not self.use_plucker: self.position_embedder = TimestepEmbedder(hidden_size, freq_type='spatial') self.angle_embedder = TimestepEmbedder(hidden_size, freq_type='angle') else: self.pose_embedder = SimpleCameraPoseEncoder(c_in=6, c_out=hidden_size) self.blocks = nn.ModuleList( [ SpatioTemporalDiTBlock( hidden_size, num_heads, mlp_ratio=mlp_ratio, is_causal=True, reference_length=reference_length, spatial_rotary_emb=self.spatial_rotary_emb, temporal_rotary_emb=self.temporal_rotary_emb, reference_rotary_emb=self.reference_rotary_emb, use_plucker=self.use_plucker, relative_embedding=relative_embedding, state_embed_only_on_qk=state_embed_only_on_qk, use_memory_attention=use_memory_attention, key_only_geometry=memory_attention_key_only_geometry, ref_mode=ref_mode ) for _ in range(depth) ] ) self.use_memory_attention = use_memory_attention self.final_layer = FinalLayer(hidden_size, patch_size, self.out_channels) self.initialize_weights() def initialize_weights(self): # Initialize transformer layers: def _basic_init(module): if isinstance(module, nn.Linear): torch.nn.init.xavier_uniform_(module.weight) if module.bias is not None: nn.init.constant_(module.bias, 0) self.apply(_basic_init) # Initialize patch_embed like nn.Linear (instead of nn.Conv2d): w = self.x_embedder.proj.weight.data nn.init.xavier_uniform_(w.view([w.shape[0], -1])) nn.init.constant_(self.x_embedder.proj.bias, 0) # Initialize timestep embedding MLP: nn.init.normal_(self.t_embedder.mlp[0].weight, std=0.02) nn.init.normal_(self.t_embedder.mlp[2].weight, std=0.02) if self.use_memory_attention: if not self.use_plucker: nn.init.normal_(self.position_embedder.mlp[0].weight, std=0.02) nn.init.normal_(self.position_embedder.mlp[2].weight, std=0.02) nn.init.normal_(self.angle_embedder.mlp[0].weight, std=0.02) nn.init.normal_(self.angle_embedder.mlp[2].weight, std=0.02) if self.add_timestamp_embedding: nn.init.normal_(self.timestamp_embedding.mlp[0].weight, std=0.02) nn.init.normal_(self.timestamp_embedding.mlp[2].weight, std=0.02) # Zero-out adaLN modulation layers in DiT blocks: for block in self.blocks: nn.init.constant_(block.s_adaLN_modulation[-1].weight, 0) nn.init.constant_(block.s_adaLN_modulation[-1].bias, 0) nn.init.constant_(block.t_adaLN_modulation[-1].weight, 0) nn.init.constant_(block.t_adaLN_modulation[-1].bias, 0) # Zero-out output layers: nn.init.constant_(self.final_layer.adaLN_modulation[-1].weight, 0) nn.init.constant_(self.final_layer.adaLN_modulation[-1].bias, 0) nn.init.constant_(self.final_layer.linear.weight, 0) nn.init.constant_(self.final_layer.linear.bias, 0) def unpatchify(self, x): """ x: (N, H, W, patch_size**2 * C) imgs: (N, H, W, C) """ c = self.out_channels p = self.x_embedder.patch_size[0] h = x.shape[1] w = x.shape[2] x = x.reshape(shape=(x.shape[0], h, w, p, p, c)) x = torch.einsum("nhwpqc->nchpwq", x) imgs = x.reshape(shape=(x.shape[0], c, h * p, w * p)) return imgs def _frame_memory_pose_to_c2w(self, pose): B, T = pose.shape[:2] x, y, z, pitch, yaw = pose.unbind(dim=-1) pitch, yaw = torch.deg2rad(pitch), torch.deg2rad(yaw) cp, sp = torch.cos(pitch), torch.sin(pitch) cy, sy = torch.cos(yaw), torch.sin(yaw) one, zero = torch.ones_like(pitch), torch.zeros_like(pitch) r_pitch = torch.stack((one, zero, zero, zero, cp, -sp, zero, sp, cp), dim=-1).reshape(B, T, 3, 3) r_yaw = torch.stack((cy, zero, sy, zero, one, zero, -sy, zero, cy), dim=-1).reshape(B, T, 3, 3) c2w = torch.eye(4, device=pose.device, dtype=pose.dtype).view(1, 1, 4, 4).repeat(B, T, 1, 1) c2w[:, :, :3, :3] = torch.matmul(r_yaw, r_pitch) c2w[:, :, :3, 3] = torch.stack((x, y, z), dim=-1) return c2w def _build_frame_memory_geometry(self, frame_memory_segments, frame_memory_masks, frame_memory_pose, image_hw, grid_h, grid_w, device, dtype, frame_idx=None): if frame_memory_segments is None: return None pose = frame_memory_pose.to(device=device) frame_idx = frame_idx.to(device=device) if torch.is_tensor(frame_idx) else None B = pose.shape[0] target_frames = int(frame_memory_segments.get("target", 0)) image_hw = image_hw if torch.is_tensor(image_hw) else torch.as_tensor(image_hw) image_hw = image_hw.to(device=device) if image_hw.ndim == 1: image_hw = image_hw.unsqueeze(0) if image_hw.shape[0] == 1 and B != 1: image_hw = image_hw.expand(B, -1) image_hw = image_hw.contiguous() # Mixed precision can round camera offsets before relative poses/rays are # formed, so keep this local geometry block in fp32 and only cast caches. with torch.autocast(device_type=device.type, enabled=False): pose = pose.to(dtype=torch.float32) image_hw = image_hw.to(dtype=torch.float32) image_h, image_w = image_hw[:, 0].view(B, 1, 1), image_hw[:, 1].view(B, 1, 1) # Token grid sets position count; image_hw keeps camera scaling in original image coordinates. y = (torch.arange(grid_h, device=device, dtype=torch.float32).view(1, grid_h, 1) + 0.5) * (image_h / grid_h) x = (torch.arange(grid_w, device=device, dtype=torch.float32).view(1, 1, grid_w) + 0.5) * (image_w / grid_w) fx, fy = self.focal_length * image_w, self.focal_length * image_h cx, cy = 0.5 * image_w, 0.5 * image_h directions = torch.stack( ( (-(x - cx) / fx).expand(-1, grid_h, -1), (-(y - cy) / fy).expand(-1, -1, grid_w), torch.ones((B, grid_h, grid_w), device=device, dtype=torch.float32), ), dim=-1, ) directions = F.normalize(directions, dim=-1) query_rays = torch.cat([torch.zeros_like(directions), directions], dim=-1)[:, None].expand(-1, target_frames, -1, -1, -1) intrinsics = torch.stack((fx[:, 0, 0], fy[:, 0, 0], cx[:, 0, 0], cy[:, 0, 0]), dim=-1) c2w = self._frame_memory_pose_to_c2w(pose) target_pose = pose[:, :target_frames] target_frame_idx = None if frame_idx is None else frame_idx[:, :target_frames] target_c2w = c2w[:, :target_frames] target_rot = target_c2w[:, :, :3, :3].transpose(-1, -2) target_t = target_c2w[:, :, :3, 3] target_w2c = torch.eye(4, device=device, dtype=torch.float32).view(1, 1, 4, 4).repeat(B, target_frames, 1, 1) target_w2c[:, :, :3, :3] = target_rot target_w2c[:, :, :3, 3] = -torch.matmul(target_rot, target_t.unsqueeze(-1)).squeeze(-1) cache = { "query_rays": query_rays.to(dtype=dtype), "target_pose": target_pose.to(dtype=dtype), "image_hw": image_hw.to(dtype=dtype), "intrinsics": intrinsics.to(dtype=dtype), } target_time = None if self.add_timestamp_embedding and target_frame_idx is not None and target_frames > 0: # target queries use dt=0, # memory keys use frame_memory_idx - target_frame_idx. target_dt = target_frame_idx.new_zeros(target_frame_idx.shape) target_time = self.timestamp_embedding(rearrange(target_dt, "b t -> (b t)")) target_time = rearrange(target_time, "(b t) d -> b t d", b=B, t=target_frames).to(dtype=dtype) cache["target_time"] = target_time cursor = target_frames for stream_name in ("anchor", "dynamic", "revisit"): stream_len = int(frame_memory_segments.get(stream_name, 0)) stream_pose = pose[:, cursor:cursor + stream_len] relative_time = None if self.add_timestamp_embedding and target_frame_idx is not None and stream_len > 0: stream_frame_idx = frame_idx[:, cursor:cursor + stream_len] relative_frame_idx = stream_frame_idx[:, None] - target_frame_idx[:, :, None] relative_time = self.timestamp_embedding(rearrange(relative_frame_idx, "b t m -> (b t m)")) relative_time = rearrange(relative_time, "(b t m) d -> b t m d", b=B, t=target_frames, m=stream_len).to(dtype=dtype) relative_c2w = torch.matmul(target_w2c[:, :, None], c2w[:, None, cursor:cursor + stream_len]) rays_d = torch.einsum("bhwj,btmij->btmhwi", directions, relative_c2w[..., :3, :3]) rays_d = F.normalize(rays_d, dim=-1) rays_o = relative_c2w[..., :3, 3][:, :, :, None, None, :].expand(-1, -1, -1, grid_h, grid_w, -1) stream_mask = None if frame_memory_masks is None or frame_memory_masks.get(stream_name) is None else frame_memory_masks[stream_name].to(device=device, dtype=torch.bool) cache[stream_name] = { "query_rays": query_rays.to(dtype=dtype), "target_time": target_time, "relative_pose": (stream_pose[:, None] - target_pose[:, :, None]).to(dtype=dtype), "relative_c2w": relative_c2w.to(dtype=dtype), "relative_rays": torch.cat([torch.linalg.cross(rays_o, rays_d, dim=-1), rays_d], dim=-1).to(dtype=dtype), "image_hw": image_hw.to(dtype=dtype), "intrinsics": intrinsics.to(dtype=dtype), "mask": stream_mask, } if relative_time is not None: cache[stream_name]["relative_time"] = relative_time cursor += stream_len return cache def forward(self, x, t, action_cond=None, pose_cond=None, current_frame=None, mode=None, reference_length=None, frame_idx=None, frame_memory_segments=None, frame_memory_masks=None, frame_memory_pose=None, image_hw=None): """ Forward pass of DiT. x: (B, T, C, H, W) tensor of spatial inputs (images or latent representations of images) t: (B, T,) tensor of diffusion timesteps """ B, T, C, H, W = x.shape # add spatial embeddings x = rearrange(x, "b t c h w -> (b t) c h w") x = self.x_embedder(x) # (B*T, C, H, W) -> (B*T, H/2, W/2, D) , C = 16, D = d_model # restore shape x = rearrange(x, "(b t) h w d -> b t h w d", t=T) # embed noise steps t = rearrange(t, "b t -> (b t)") c_t = self.t_embedder(t) # (N, D) c = c_t.clone() c = rearrange(c, "(b t) d -> b t d", t=T) if torch.is_tensor(action_cond): c_action_cond = c + self.external_cond(action_cond) else: c_action_cond = None if torch.is_tensor(pose_cond): if not self.use_plucker: pose_cond = pose_cond.to(action_cond.dtype) b_, t_, d_ = pose_cond.shape pos_emb = self.position_embedder(rearrange(pose_cond[...,:3], "b t d -> (b t d)")) angle_emb = self.angle_embedder(rearrange(pose_cond[...,3:], "b t d -> (b t d)")) pos_emb = rearrange(pos_emb, "(b t d) c -> b t d c", b=b_, t=t_, d=3).sum(-2) angle_emb = rearrange(angle_emb, "(b t d) c -> b t d c", b=b_, t=t_, d=2).sum(-2) pc = pos_emb + angle_emb else: pose_cond = pose_cond[:, :, ::40, ::40] # pc = self.pose_embedder(pose_cond)[0] # pc = pc.permute(0,2,3,4,1) pc = self.pose_embedder(pose_cond) pc = pc.permute(1,0,2,3,4) if torch.is_tensor(frame_idx) and self.add_timestamp_embedding: bb = frame_idx.shape[1] frame_time = rearrange(frame_idx, "t b -> (b t)") frame_time = self.timestamp_embedding(frame_time) frame_time = rearrange(frame_time, "(b t) d -> b t d", b=bb) pc = pc + frame_time[:, :, None, None] # pc = pc + rearrange(c_t.clone(), "(b t) d -> b t d", t=T)[:,:,None,None] # add time condition for different timestep scaling else: pc = None frame_memory_geometry = None if self.use_memory_attention and self.use_plucker: frame_memory_geometry = self._build_frame_memory_geometry( frame_memory_segments, frame_memory_masks, frame_memory_pose, image_hw, x.shape[2], x.shape[3], x.device, x.dtype, frame_idx=frame_idx, ) for i, block in enumerate(self.blocks): x = block(x, c, current_frame=current_frame, timestep=t, is_last_block= (i+1 == len(self.blocks)), pose_cond=pc, mode=mode, c_action_cond=c_action_cond, reference_length=reference_length, frame_memory_segments=frame_memory_segments, frame_memory_masks=frame_memory_masks, frame_memory_pose=frame_memory_pose, image_hw=image_hw, frame_memory_geometry=frame_memory_geometry) # (N, T, H, W, D) if frame_memory_segments is not None: target_frames = int(frame_memory_segments.get("target", 0)) x = x[:, :target_frames] c = c[:, :target_frames] T = target_frames x = self.final_layer(x, c) # (N, T, H, W, patch_size ** 2 * out_channels) # unpatchify x = rearrange(x, "b t h w d -> (b t) h w d") x = self.unpatchify(x) # (N, out_channels, H, W) x = rearrange(x, "(b t) c h w -> b t c h w", t=T) return x def DiT_S_2(action_cond_dim, pose_cond_dim, reference_length, use_plucker, relative_embedding, state_embed_only_on_qk, use_memory_attention, add_timestamp_embedding, ref_mode, focal_length=0.35, memory_attention_key_only_geometry=True): return DiT( patch_size=2, hidden_size=1024, depth=16, num_heads=16, action_cond_dim=action_cond_dim, pose_cond_dim=pose_cond_dim, reference_length=reference_length, use_plucker=use_plucker, relative_embedding=relative_embedding, state_embed_only_on_qk=state_embed_only_on_qk, use_memory_attention=use_memory_attention, add_timestamp_embedding=add_timestamp_embedding, memory_attention_key_only_geometry=memory_attention_key_only_geometry, ref_mode=ref_mode, focal_length=focal_length, ) DiT_models = {"DiT-S/2": DiT_S_2}