algorithms/dememwm/models/dit.py

"""
References:
    - DiT: https://github.com/facebookresearch/DiT/blob/main/models.py
    - Diffusion Forcing: https://github.com/buoyancy99/diffusion-forcing/blob/main/algorithms/diffusion_forcing/models/unet3d.py
    - Latte: https://github.com/Vchitect/Latte/blob/main/models/latte.py
"""

from typing import Optional, Literal
import torch
from torch import nn
from torch.nn import functional as F
from .rotary_embedding_torch import RotaryEmbedding
from einops import rearrange
from .attention import SpatialAxialAttention, TemporalAxialAttention
from timm.models.vision_transformer import Mlp
from timm.layers.helpers import to_2tuple
import math
from collections import namedtuple
from typing import Optional, Callable
from .cameractrl_module import SimpleCameraPoseEncoder


def modulate(x, shift, scale):
    fixed_dims = [1] * len(shift.shape[1:])
    shift = shift.repeat(x.shape[0] // shift.shape[0], *fixed_dims)
    scale = scale.repeat(x.shape[0] // scale.shape[0], *fixed_dims)
    while shift.dim() < x.dim():
        shift = shift.unsqueeze(-2)
        scale = scale.unsqueeze(-2)
    return x * (1 + scale) + shift

def gate(x, g):
    fixed_dims = [1] * len(g.shape[1:])
    g = g.repeat(x.shape[0] // g.shape[0], *fixed_dims)
    while g.dim() < x.dim():
        g = g.unsqueeze(-2)
    return g * x


class PatchEmbed(nn.Module):
    """2D Image to Patch Embedding"""

    def __init__(
        self,
        img_height=256,
        img_width=256,
        patch_size=16,
        in_chans=3,
        embed_dim=768,
        norm_layer=None,
        flatten=True,
    ):
        super().__init__()
        img_size = (img_height, img_width)
        patch_size = to_2tuple(patch_size)
        self.img_size = img_size
        self.patch_size = patch_size
        self.grid_size = (img_size[0] // patch_size[0], img_size[1] // patch_size[1])
        self.num_patches = self.grid_size[0] * self.grid_size[1]
        self.flatten = flatten

        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)
        self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()

    def forward(self, x, random_sample=False):
        B, C, H, W = x.shape
        assert random_sample or (H == self.img_size[0] and W == self.img_size[1]), f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})."
        
        x = self.proj(x)
        if self.flatten:
            x = rearrange(x, "B C H W -> B (H W) C")
        else:
            x = rearrange(x, "B C H W -> B H W C")
        x = self.norm(x)
        return x


class TimestepEmbedder(nn.Module):
    """
    Embeds scalar timesteps into vector representations.
    """

    def __init__(self, hidden_size, frequency_embedding_size=256, freq_type='time_step'):
        super().__init__()
        self.mlp = nn.Sequential(
            nn.Linear(frequency_embedding_size, hidden_size, bias=True),  # hidden_size is diffusion model hidden size
            nn.SiLU(),
            nn.Linear(hidden_size, hidden_size, bias=True),
        )
        self.frequency_embedding_size = frequency_embedding_size
        self.freq_type = freq_type

    @staticmethod
    def timestep_embedding(t, dim, max_period=10000, freq_type='time_step'):
        """
        Create sinusoidal timestep embeddings.
        :param t: a 1-D Tensor of N indices, one per batch element.
                          These may be fractional.
        :param dim: the dimension of the output.
        :param max_period: controls the minimum frequency of the embeddings.
        :return: an (N, D) Tensor of positional embeddings.
        """
        # https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.py
        half = dim // 2

        if freq_type == 'time_step':
            freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half).to(device=t.device)
        elif freq_type == 'spatial': # ~(-5 5)
            freqs = torch.linspace(1.0, half, half).to(device=t.device) * torch.pi
        elif freq_type == 'angle': # 0-360
            freqs = torch.linspace(1.0, half, half).to(device=t.device) * torch.pi / 180


        args = t[:, None].float() * freqs[None]
        
        embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
        if dim % 2:
            embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)
        return embedding

    def forward(self, t):
        t_freq = self.timestep_embedding(t, self.frequency_embedding_size, freq_type=self.freq_type)
        t_emb = self.mlp(t_freq)
        return t_emb


class FinalLayer(nn.Module):
    """
    The final layer of DiT.
    """

    def __init__(self, hidden_size, patch_size, out_channels):
        super().__init__()
        self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
        self.linear = nn.Linear(hidden_size, patch_size * patch_size * out_channels, bias=True)
        self.adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=True))

    def forward(self, x, c):
        shift, scale = self.adaLN_modulation(c).chunk(2, dim=-1)
        x = modulate(self.norm_final(x), shift, scale)
        x = self.linear(x)
        return x


class FrameMemoryReferenceAttention(nn.Module):
    def __init__(self, hidden_size, num_heads, key_only_geometry=True):
        super().__init__()
        self.hidden_size = hidden_size
        self.num_heads = num_heads
        self.head_dim = hidden_size // num_heads
        self.key_only_geometry = key_only_geometry
        self.to_q = nn.Linear(hidden_size, hidden_size, bias=False)
        self.to_k = nn.Linear(hidden_size, hidden_size, bias=False)
        self.to_v = nn.Linear(hidden_size, hidden_size, bias=False)
        self.query_pose_proj = nn.Linear(6, hidden_size, bias=True)
        self.key_pose_proj = nn.Linear(6, hidden_size, bias=True)
        self.out_proj = nn.Linear(hidden_size, hidden_size, bias=True)

    def _split_heads(self, x):
        return rearrange(x, "b n (h d) -> b h n d", h=self.num_heads)


    def forward(self, target_hidden, memory_hidden, memory_mask=None, geometry_cache=None):
        B, T_target, H, W, D = target_hidden.shape
        if memory_hidden is None or int(memory_hidden.shape[1]) == 0:
            return target_hidden.new_zeros(target_hidden.shape)

        T_memory = memory_hidden.shape[1]
        P = H * W
        if geometry_cache is not None:
            query_rays = None if self.key_only_geometry else geometry_cache.get("query_rays")
            if query_rays is not None:
                query_rays = query_rays.to(device=target_hidden.device, dtype=target_hidden.dtype)
            relative_rays = geometry_cache.get("relative_rays")
            if relative_rays is not None:
                relative_rays = relative_rays.to(device=target_hidden.device, dtype=target_hidden.dtype)
            target_time = None if self.key_only_geometry else geometry_cache.get("target_time")
            relative_time = geometry_cache.get("relative_time")
        else:
            query_rays = None
            relative_rays = None
            target_time = None
            relative_time = None

        q_input = target_hidden
        if query_rays is not None:
            # Values stay visual-only; geometry and time only steer Q/K matching.
            q_input = q_input + self.query_pose_proj(query_rays)
        if target_time is not None:
            q_input = q_input + target_time.to(device=target_hidden.device, dtype=target_hidden.dtype)[:, :, None, None]
        q = rearrange(q_input, "b t h w d -> (b t) (h w) d")
        q = self._split_heads(self.to_q(q))

        memory_tokens = rearrange(memory_hidden, "b m h w d -> b m (h w) d")
        memory_flat = rearrange(memory_tokens, "b m p d -> b (m p) d")
        key_cond = None
        if relative_rays is not None:
            key_cond = self.key_pose_proj(
                rearrange(relative_rays, "b t m h w r -> b t m (h w) r")
            )
        if relative_time is not None:
            relative_time = relative_time.to(device=target_hidden.device, dtype=target_hidden.dtype)
            relative_time = relative_time[:, :, :, None]
            key_cond = relative_time if key_cond is None else key_cond + relative_time
        if key_cond is None:
            k = self._split_heads(self.to_k(memory_flat))
        else:
            k = self.to_k(memory_tokens[:, None] + key_cond)
            k = self._split_heads(rearrange(k, "b t m p d -> (b t) (m p) d"))
        v = self._split_heads(self.to_v(memory_flat))

        if memory_mask is None:
            frame_valid = torch.ones((B, T_memory), device=target_hidden.device, dtype=torch.bool)
        else:
            frame_valid = memory_mask.to(device=target_hidden.device, dtype=torch.bool)
        key_valid = frame_valid[:, :, None].expand(B, T_memory, P).reshape(B, T_memory * P)
        active = key_valid.any(dim=1)
        key_valid = torch.cat([key_valid, ~active[:, None]], dim=1)

        row_batch = torch.arange(B, device=target_hidden.device).repeat_interleave(T_target)
        if key_cond is None:
            k = k.index_select(0, row_batch)
        # A zero dummy key keeps all-padded samples finite; the projected delta
        # is masked back to zero below so padded memory cannot contribute.
        dummy_k = k.new_zeros((k.shape[0], self.num_heads, 1, self.head_dim))
        k = torch.cat([k, dummy_k], dim=2)
        dummy_v = v.new_zeros((B, self.num_heads, 1, self.head_dim))
        v = torch.cat([v, dummy_v], dim=2).index_select(0, row_batch)

        attn_bias = torch.zeros((B * T_target, 1, 1, key_valid.shape[1]), device=q.device, dtype=q.dtype)
        attn_bias = attn_bias.masked_fill(~key_valid[row_batch][:, None, None], float("-inf"))

        x = F.scaled_dot_product_attention(
            query=q.contiguous(),
            key=k.contiguous(),
            value=v.contiguous(),
            attn_mask=attn_bias,
        )
        x = rearrange(x, "n h p d -> n p (h d)")
        x = self.out_proj(x).to(dtype=target_hidden.dtype)
        x = rearrange(x, "(b t) (h w) d -> b t h w d", b=B, t=T_target, h=H, w=W)
        return x * active[:, None, None, None, None].to(dtype=x.dtype)


class SpatioTemporalDiTBlock(nn.Module):
    def __init__(
        self,
        hidden_size,
        num_heads,
        reference_length,
        mlp_ratio=4.0,
        is_causal=True,
        spatial_rotary_emb: Optional[RotaryEmbedding] = None,
        temporal_rotary_emb: Optional[RotaryEmbedding] = None,
        reference_rotary_emb=None,
        use_plucker=False,
        relative_embedding=False,
        state_embed_only_on_qk=False,
        use_memory_attention=False,
        key_only_geometry=True,
        ref_mode='sequential'
    ):
        super().__init__()
        self.is_causal = is_causal
        mlp_hidden_dim = int(hidden_size * mlp_ratio)
        approx_gelu = lambda: nn.GELU(approximate="tanh")

        self.s_norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
        self.s_attn = SpatialAxialAttention(
            hidden_size,
            heads=num_heads,
            dim_head=hidden_size // num_heads,
            rotary_emb=spatial_rotary_emb
        )
        self.s_norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
        self.s_mlp = Mlp(
            in_features=hidden_size,
            hidden_features=mlp_hidden_dim,
            act_layer=approx_gelu,
            drop=0,
        )
        self.s_adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 6 * hidden_size, bias=True))

        self.t_norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
        self.t_attn = TemporalAxialAttention(
            hidden_size,
            heads=num_heads,
            dim_head=hidden_size // num_heads,
            is_causal=is_causal,
            rotary_emb=temporal_rotary_emb,
            reference_length=reference_length
        )
        self.t_norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
        self.t_mlp = Mlp(
            in_features=hidden_size,
            hidden_features=mlp_hidden_dim,
            act_layer=approx_gelu,
            drop=0,
        )
        self.t_adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 6 * hidden_size, bias=True))

        self.use_memory_attention = use_memory_attention
        if self.use_memory_attention:
            self.r_norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
            self.r_norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
            self.r_mlp = Mlp(
                in_features=hidden_size,
                hidden_features=mlp_hidden_dim,
                act_layer=approx_gelu,
                drop=0,
            )
            self.r_adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 6 * hidden_size, bias=True))
            self.r_attn_anchor = FrameMemoryReferenceAttention(hidden_size, num_heads, key_only_geometry=key_only_geometry)
            self.r_attn_dynamic = FrameMemoryReferenceAttention(hidden_size, num_heads, key_only_geometry=key_only_geometry)
            self.r_attn_revisit = FrameMemoryReferenceAttention(hidden_size, num_heads, key_only_geometry=key_only_geometry)

        self.reference_length = reference_length
        self.relative_embedding = relative_embedding
        self.state_embed_only_on_qk = state_embed_only_on_qk

        self.ref_mode = ref_mode

        if self.ref_mode == 'parallel':
            self.parallel_map = nn.Linear(hidden_size, hidden_size)

    def _split_frame_memory(self, x, frame_memory_segments):
        target = int(frame_memory_segments.get("target", 0))
        anchor = int(frame_memory_segments.get("anchor", 0))
        dynamic = int(frame_memory_segments.get("dynamic", 0))
        revisit = int(frame_memory_segments.get("revisit", 0))
        if min(target, anchor, dynamic, revisit) < 0:
            raise ValueError(
                "frame_memory_segments lengths must be nonnegative; "
                f"got target={target}, anchor={anchor}, dynamic={dynamic}, revisit={revisit}"
            )
        total = target + anchor + dynamic + revisit
        if total != int(x.shape[1]):
            raise ValueError(
                f"frame_memory_segments lengths sum to {total}, expected x.shape[1]={int(x.shape[1])}"
            )
        a0, a1 = target, target + anchor
        d0, d1 = a1, a1 + dynamic
        r0, r1 = d1, d1 + revisit
        return x[:, :target], x[:, a0:a1], x[:, d0:d1], x[:, r0:r1]

    def _frame_memory_stream_mask(self, frame_memory_masks, stream_name, stream_hidden):
        self._check_frame_memory_stream_mask_shape(frame_memory_masks, stream_name, stream_hidden)
        if stream_hidden is None or int(stream_hidden.shape[1]) == 0:
            return None
        if frame_memory_masks is None or frame_memory_masks.get(stream_name) is None:
            return None
        return frame_memory_masks[stream_name].to(device=stream_hidden.device, dtype=torch.bool)

    def _check_frame_memory_stream_mask_shape(self, frame_memory_masks, stream_name, stream_hidden):
        if stream_hidden is None:
            return
        if frame_memory_masks is None or frame_memory_masks.get(stream_name) is None:
            return
        stream_mask = frame_memory_masks[stream_name]
        expected_shape = (int(stream_hidden.shape[0]), int(stream_hidden.shape[1]))
        if tuple(stream_mask.shape) != expected_shape:
            raise ValueError(
                f"frame_memory_masks[{stream_name!r}] shape {tuple(stream_mask.shape)} "
                f"must match {expected_shape}"
            )

    def _apply_frame_memory_reference_attention(self, x, c, frame_memory_segments, frame_memory_masks, frame_memory_geometry=None):
        x_target, x_anchor, x_dynamic, x_revisit = self._split_frame_memory(x, frame_memory_segments)
        if int(x_target.shape[1]) == 0:
            return x

        r_shift_msa, r_scale_msa, r_gate_msa, r_shift_mlp, r_scale_mlp, r_gate_mlp = self.r_adaLN_modulation(c).chunk(6, dim=-1)
        attn_input = modulate(self.r_norm1(x), r_shift_msa, r_scale_msa)
        attn_target, attn_anchor, attn_dynamic, attn_revisit = self._split_frame_memory(attn_input, frame_memory_segments)

        deltas = []
        active_stream_count = x_target.new_zeros((x_target.shape[0],))
        for stream_name, r_attn, stream_hidden, stream_attn in (
            ("anchor", self.r_attn_anchor, x_anchor, attn_anchor),
            ("dynamic", self.r_attn_dynamic, x_dynamic, attn_dynamic),
            ("revisit", self.r_attn_revisit, x_revisit, attn_revisit),
        ):
            if int(stream_hidden.shape[1]) == 0:
                continue
            stream_mask = self._frame_memory_stream_mask(frame_memory_masks, stream_name, stream_hidden)
            stream_geometry = None if frame_memory_geometry is None else frame_memory_geometry.get(stream_name)
            deltas.append(r_attn(attn_target, stream_attn, stream_mask, geometry_cache=stream_geometry))
            if stream_mask is None:
                active_stream_count = active_stream_count + 1
            else:
                active_stream_count = active_stream_count + stream_mask.any(dim=1).to(dtype=active_stream_count.dtype)

        if deltas:
            # Average over streams that have at least one valid memory frame per
            # sample; samples with no active streams receive a zero reference delta.
            r_delta = sum(deltas)
            active = active_stream_count > 0
            r_delta = r_delta / active_stream_count.clamp_min(1)[:, None, None, None, None]
            r_delta = r_delta * active[:, None, None, None, None].to(dtype=r_delta.dtype)
            x_target = x_target + gate(r_delta, r_gate_msa[:, :x_target.shape[1]])

        x = torch.cat([x_target, x_anchor, x_dynamic, x_revisit], dim=1)
        x = x + gate(self.r_mlp(modulate(self.r_norm2(x), r_shift_mlp, r_scale_mlp)), r_gate_mlp)
        return x

    def forward(self, x, c, current_frame=None, timestep=None, is_last_block=False, 
        pose_cond=None, mode="training", c_action_cond=None, reference_length=None,
        frame_memory_segments=None, frame_memory_masks=None, frame_memory_pose=None,
        image_hw=None, frame_memory_geometry=None):
        B, T, H, W, D = x.shape
        if frame_memory_segments is not None:
            for stream_name, stream_hidden in zip(
                ("target", "anchor", "dynamic", "revisit"),
                self._split_frame_memory(x, frame_memory_segments),
            ):
                self._check_frame_memory_stream_mask_shape(frame_memory_masks, stream_name, stream_hidden)

        # spatial block
        
        s_shift_msa, s_scale_msa, s_gate_msa, s_shift_mlp, s_scale_mlp, s_gate_mlp = self.s_adaLN_modulation(c).chunk(6, dim=-1)
        x = x + gate(self.s_attn(modulate(self.s_norm1(x), s_shift_msa, s_scale_msa)), s_gate_msa)
        x = x + gate(self.s_mlp(modulate(self.s_norm2(x), s_shift_mlp, s_scale_mlp)), s_gate_mlp)

        # temporal block
        if c_action_cond is not None:
            t_shift_msa, t_scale_msa, t_gate_msa, t_shift_mlp, t_scale_mlp, t_gate_mlp = self.t_adaLN_modulation(c_action_cond).chunk(6, dim=-1)
        else:
            t_shift_msa, t_scale_msa, t_gate_msa, t_shift_mlp, t_scale_mlp, t_gate_mlp = self.t_adaLN_modulation(c).chunk(6, dim=-1)
        
        x_t = x + gate(self.t_attn(
            modulate(self.t_norm1(x), t_shift_msa, t_scale_msa),
            frame_memory_segments=frame_memory_segments,
            frame_memory_masks=frame_memory_masks,
        ), t_gate_msa)
        x_t = x_t + gate(self.t_mlp(modulate(self.t_norm2(x_t), t_shift_mlp, t_scale_mlp)), t_gate_mlp)

        if self.ref_mode == 'sequential':
            x = x_t

        # memory block
        if self.use_memory_attention and frame_memory_segments is not None:
            x = self._apply_frame_memory_reference_attention(
                x,
                c,
                frame_memory_segments,
                frame_memory_masks,
                frame_memory_geometry,
            )

        if self.ref_mode == 'parallel':
            x = x_t + self.parallel_map(x)

        return x


class DiT(nn.Module):
    """
    Diffusion model with a Transformer backbone.
    """

    def __init__(
        self,
        input_h=18,
        input_w=32,
        patch_size=2,
        in_channels=16,
        hidden_size=1024,
        depth=12,
        num_heads=16,
        mlp_ratio=4.0,
        action_cond_dim=25,
        pose_cond_dim=4,
        max_frames=32,
        reference_length=8,
        use_plucker=False,
        relative_embedding=False,
        state_embed_only_on_qk=False,
        use_memory_attention=False,
        add_timestamp_embedding=False,
        ref_mode='sequential',
        focal_length=0.35,
        memory_attention_key_only_geometry=True,
    ):
        super().__init__()
        self.in_channels = in_channels
        self.out_channels = in_channels
        self.patch_size = patch_size
        self.num_heads = num_heads
        self.max_frames = max_frames
        self.focal_length = focal_length

        self.x_embedder = PatchEmbed(input_h, input_w, patch_size, in_channels, hidden_size, flatten=False)
        self.t_embedder = TimestepEmbedder(hidden_size)

        self.add_timestamp_embedding = add_timestamp_embedding
        if self.add_timestamp_embedding:
            self.timestamp_embedding = TimestepEmbedder(hidden_size)

        frame_h, frame_w = self.x_embedder.grid_size

        self.spatial_rotary_emb = RotaryEmbedding(dim=hidden_size // num_heads // 2, freqs_for="pixel", max_freq=256)
        self.temporal_rotary_emb = RotaryEmbedding(dim=hidden_size // num_heads)
        # self.reference_rotary_emb = RotaryEmbedding(dim=hidden_size // num_heads // 2, freqs_for="pixel", max_freq=256)
        self.reference_rotary_emb = None

        self.external_cond = nn.Linear(action_cond_dim, hidden_size) if action_cond_dim > 0 else nn.Identity()

        # self.pose_cond = nn.Linear(pose_cond_dim, hidden_size) if pose_cond_dim > 0 else nn.Identity()
        
        self.use_plucker = use_plucker
        if not self.use_plucker:
            self.position_embedder = TimestepEmbedder(hidden_size, freq_type='spatial')
            self.angle_embedder = TimestepEmbedder(hidden_size, freq_type='angle')
        else:
            self.pose_embedder = SimpleCameraPoseEncoder(c_in=6, c_out=hidden_size)

        self.blocks = nn.ModuleList(
            [
                SpatioTemporalDiTBlock(
                    hidden_size,
                    num_heads,
                    mlp_ratio=mlp_ratio,
                    is_causal=True,
                    reference_length=reference_length,
                    spatial_rotary_emb=self.spatial_rotary_emb,
                    temporal_rotary_emb=self.temporal_rotary_emb,
                    reference_rotary_emb=self.reference_rotary_emb,
                    use_plucker=self.use_plucker,
                    relative_embedding=relative_embedding,
                    state_embed_only_on_qk=state_embed_only_on_qk,
                    use_memory_attention=use_memory_attention,
                    key_only_geometry=memory_attention_key_only_geometry,
                    ref_mode=ref_mode
                )
                for _ in range(depth)
            ]
        )
        self.use_memory_attention = use_memory_attention
        self.final_layer = FinalLayer(hidden_size, patch_size, self.out_channels)
        self.initialize_weights()

    def initialize_weights(self):
        # Initialize transformer layers:
        def _basic_init(module):
            if isinstance(module, nn.Linear):
                torch.nn.init.xavier_uniform_(module.weight)
                if module.bias is not None:
                    nn.init.constant_(module.bias, 0)

        self.apply(_basic_init)

        # Initialize patch_embed like nn.Linear (instead of nn.Conv2d):
        w = self.x_embedder.proj.weight.data
        nn.init.xavier_uniform_(w.view([w.shape[0], -1]))
        nn.init.constant_(self.x_embedder.proj.bias, 0)

        # Initialize timestep embedding MLP:
        nn.init.normal_(self.t_embedder.mlp[0].weight, std=0.02)
        nn.init.normal_(self.t_embedder.mlp[2].weight, std=0.02)

        if self.use_memory_attention:
            if not self.use_plucker:
                nn.init.normal_(self.position_embedder.mlp[0].weight, std=0.02)
                nn.init.normal_(self.position_embedder.mlp[2].weight, std=0.02)

                nn.init.normal_(self.angle_embedder.mlp[0].weight, std=0.02)
                nn.init.normal_(self.angle_embedder.mlp[2].weight, std=0.02)
            
            if self.add_timestamp_embedding:
                nn.init.normal_(self.timestamp_embedding.mlp[0].weight, std=0.02)
                nn.init.normal_(self.timestamp_embedding.mlp[2].weight, std=0.02)


        # Zero-out adaLN modulation layers in DiT blocks:
        for block in self.blocks:
            nn.init.constant_(block.s_adaLN_modulation[-1].weight, 0)
            nn.init.constant_(block.s_adaLN_modulation[-1].bias, 0)
            nn.init.constant_(block.t_adaLN_modulation[-1].weight, 0)
            nn.init.constant_(block.t_adaLN_modulation[-1].bias, 0)

        # Zero-out output layers:
        nn.init.constant_(self.final_layer.adaLN_modulation[-1].weight, 0)
        nn.init.constant_(self.final_layer.adaLN_modulation[-1].bias, 0)
        nn.init.constant_(self.final_layer.linear.weight, 0)
        nn.init.constant_(self.final_layer.linear.bias, 0)

    def unpatchify(self, x):
        """
        x: (N, H, W, patch_size**2 * C)
        imgs: (N, H, W, C)
        """
        c = self.out_channels
        p = self.x_embedder.patch_size[0]
        h = x.shape[1]
        w = x.shape[2]

        x = x.reshape(shape=(x.shape[0], h, w, p, p, c))
        x = torch.einsum("nhwpqc->nchpwq", x)
        imgs = x.reshape(shape=(x.shape[0], c, h * p, w * p))
        return imgs

    def _frame_memory_pose_to_c2w(self, pose):
        B, T = pose.shape[:2]
        x, y, z, pitch, yaw = pose.unbind(dim=-1)
        pitch, yaw = torch.deg2rad(pitch), torch.deg2rad(yaw)
        cp, sp = torch.cos(pitch), torch.sin(pitch)
        cy, sy = torch.cos(yaw), torch.sin(yaw)
        one, zero = torch.ones_like(pitch), torch.zeros_like(pitch)
        r_pitch = torch.stack((one, zero, zero, zero, cp, -sp, zero, sp, cp), dim=-1).reshape(B, T, 3, 3)
        r_yaw = torch.stack((cy, zero, sy, zero, one, zero, -sy, zero, cy), dim=-1).reshape(B, T, 3, 3)
        c2w = torch.eye(4, device=pose.device, dtype=pose.dtype).view(1, 1, 4, 4).repeat(B, T, 1, 1)
        c2w[:, :, :3, :3] = torch.matmul(r_yaw, r_pitch)
        c2w[:, :, :3, 3] = torch.stack((x, y, z), dim=-1)
        return c2w

    def _build_frame_memory_geometry(self, frame_memory_segments, frame_memory_masks, frame_memory_pose, image_hw, grid_h, grid_w, device, dtype, frame_idx=None):
        if frame_memory_segments is None:
            return None

        pose = frame_memory_pose.to(device=device)
        frame_idx = frame_idx.to(device=device) if torch.is_tensor(frame_idx) else None
        B = pose.shape[0]
        target_frames = int(frame_memory_segments.get("target", 0))
        image_hw = image_hw if torch.is_tensor(image_hw) else torch.as_tensor(image_hw)
        image_hw = image_hw.to(device=device)
        if image_hw.ndim == 1:
            image_hw = image_hw.unsqueeze(0)
        if image_hw.shape[0] == 1 and B != 1:
            image_hw = image_hw.expand(B, -1)
        image_hw = image_hw.contiguous()
        # Mixed precision can round camera offsets before relative poses/rays are
        # formed, so keep this local geometry block in fp32 and only cast caches.
        with torch.autocast(device_type=device.type, enabled=False):
            pose = pose.to(dtype=torch.float32)
            image_hw = image_hw.to(dtype=torch.float32)
            image_h, image_w = image_hw[:, 0].view(B, 1, 1), image_hw[:, 1].view(B, 1, 1)

            # Token grid sets position count; image_hw keeps camera scaling in original image coordinates.
            y = (torch.arange(grid_h, device=device, dtype=torch.float32).view(1, grid_h, 1) + 0.5) * (image_h / grid_h)
            x = (torch.arange(grid_w, device=device, dtype=torch.float32).view(1, 1, grid_w) + 0.5) * (image_w / grid_w)
            fx, fy = self.focal_length * image_w, self.focal_length * image_h
            cx, cy = 0.5 * image_w, 0.5 * image_h
            directions = torch.stack(
                (
                    (-(x - cx) / fx).expand(-1, grid_h, -1),
                    (-(y - cy) / fy).expand(-1, -1, grid_w),
                    torch.ones((B, grid_h, grid_w), device=device, dtype=torch.float32),
                ),
                dim=-1,
            )
            directions = F.normalize(directions, dim=-1)
            query_rays = torch.cat([torch.zeros_like(directions), directions], dim=-1)[:, None].expand(-1, target_frames, -1, -1, -1)
            intrinsics = torch.stack((fx[:, 0, 0], fy[:, 0, 0], cx[:, 0, 0], cy[:, 0, 0]), dim=-1)

            c2w = self._frame_memory_pose_to_c2w(pose)
            target_pose = pose[:, :target_frames]
            target_frame_idx = None if frame_idx is None else frame_idx[:, :target_frames]
            target_c2w = c2w[:, :target_frames]
            target_rot = target_c2w[:, :, :3, :3].transpose(-1, -2)
            target_t = target_c2w[:, :, :3, 3]
            target_w2c = torch.eye(4, device=device, dtype=torch.float32).view(1, 1, 4, 4).repeat(B, target_frames, 1, 1)
            target_w2c[:, :, :3, :3] = target_rot
            target_w2c[:, :, :3, 3] = -torch.matmul(target_rot, target_t.unsqueeze(-1)).squeeze(-1)

            cache = {
                "query_rays": query_rays.to(dtype=dtype),
                "target_pose": target_pose.to(dtype=dtype),
                "image_hw": image_hw.to(dtype=dtype),
                "intrinsics": intrinsics.to(dtype=dtype),
            }
            target_time = None
            if self.add_timestamp_embedding and target_frame_idx is not None and target_frames > 0:
                # target queries use dt=0,
                # memory keys use frame_memory_idx - target_frame_idx.
                target_dt = target_frame_idx.new_zeros(target_frame_idx.shape)
                target_time = self.timestamp_embedding(rearrange(target_dt, "b t -> (b t)"))
                target_time = rearrange(target_time, "(b t) d -> b t d", b=B, t=target_frames).to(dtype=dtype)
                cache["target_time"] = target_time
            cursor = target_frames
            for stream_name in ("anchor", "dynamic", "revisit"):
                stream_len = int(frame_memory_segments.get(stream_name, 0))
                stream_pose = pose[:, cursor:cursor + stream_len]
                relative_time = None
                if self.add_timestamp_embedding and target_frame_idx is not None and stream_len > 0:
                    stream_frame_idx = frame_idx[:, cursor:cursor + stream_len]
                    relative_frame_idx = stream_frame_idx[:, None] - target_frame_idx[:, :, None]
                    relative_time = self.timestamp_embedding(rearrange(relative_frame_idx, "b t m -> (b t m)"))
                    relative_time = rearrange(relative_time, "(b t m) d -> b t m d", b=B, t=target_frames, m=stream_len).to(dtype=dtype)
                relative_c2w = torch.matmul(target_w2c[:, :, None], c2w[:, None, cursor:cursor + stream_len])
                rays_d = torch.einsum("bhwj,btmij->btmhwi", directions, relative_c2w[..., :3, :3])
                rays_d = F.normalize(rays_d, dim=-1)
                rays_o = relative_c2w[..., :3, 3][:, :, :, None, None, :].expand(-1, -1, -1, grid_h, grid_w, -1)
                stream_mask = None if frame_memory_masks is None or frame_memory_masks.get(stream_name) is None else frame_memory_masks[stream_name].to(device=device, dtype=torch.bool)
                cache[stream_name] = {
                    "query_rays": query_rays.to(dtype=dtype),
                    "target_time": target_time,
                    "relative_pose": (stream_pose[:, None] - target_pose[:, :, None]).to(dtype=dtype),
                    "relative_c2w": relative_c2w.to(dtype=dtype),
                    "relative_rays": torch.cat([torch.linalg.cross(rays_o, rays_d, dim=-1), rays_d], dim=-1).to(dtype=dtype),
                    "image_hw": image_hw.to(dtype=dtype),
                    "intrinsics": intrinsics.to(dtype=dtype),
                    "mask": stream_mask,
                }
                if relative_time is not None:
                    cache[stream_name]["relative_time"] = relative_time
                cursor += stream_len
        return cache

    def forward(self, x, t, action_cond=None, pose_cond=None, current_frame=None, mode=None, 
                    reference_length=None, frame_idx=None, frame_memory_segments=None,
                    frame_memory_masks=None, frame_memory_pose=None, image_hw=None):
        """
        Forward pass of DiT.
        x: (B, T, C, H, W) tensor of spatial inputs (images or latent representations of images)
        t: (B, T,) tensor of diffusion timesteps
        """

        B, T, C, H, W = x.shape

        # add spatial embeddings
        x = rearrange(x, "b t c h w -> (b t) c h w")

        x = self.x_embedder(x)  # (B*T, C, H, W) -> (B*T, H/2, W/2, D) , C = 16, D = d_model
        # restore shape
        x = rearrange(x, "(b t) h w d -> b t h w d", t=T)
        # embed noise steps
        t = rearrange(t, "b t -> (b t)")

        c_t = self.t_embedder(t)  # (N, D)
        c = c_t.clone()
        c = rearrange(c, "(b t) d -> b t d", t=T)

        if torch.is_tensor(action_cond):
            c_action_cond = c + self.external_cond(action_cond)
        else:
            c_action_cond = None
        
        if torch.is_tensor(pose_cond):
            if not self.use_plucker:
                pose_cond = pose_cond.to(action_cond.dtype)
                b_, t_, d_ = pose_cond.shape
                pos_emb = self.position_embedder(rearrange(pose_cond[...,:3], "b t d -> (b t d)"))
                angle_emb = self.angle_embedder(rearrange(pose_cond[...,3:], "b t d -> (b t d)"))
                pos_emb = rearrange(pos_emb, "(b t d) c -> b t d c", b=b_, t=t_, d=3).sum(-2)
                angle_emb = rearrange(angle_emb, "(b t d) c -> b t d c", b=b_, t=t_, d=2).sum(-2)
                pc = pos_emb + angle_emb
            else:
                pose_cond = pose_cond[:, :, ::40, ::40]
                # pc = self.pose_embedder(pose_cond)[0]
                # pc = pc.permute(0,2,3,4,1)
                pc = self.pose_embedder(pose_cond)
                pc = pc.permute(1,0,2,3,4)

                if torch.is_tensor(frame_idx) and self.add_timestamp_embedding:
                    bb = frame_idx.shape[1]
                    frame_time = rearrange(frame_idx, "t b -> (b t)")
                    frame_time = self.timestamp_embedding(frame_time)
                    frame_time = rearrange(frame_time, "(b t) d -> b t d", b=bb)
                    pc = pc + frame_time[:, :, None, None]

                # pc = pc + rearrange(c_t.clone(), "(b t) d -> b t d", t=T)[:,:,None,None] # add time condition for different timestep scaling 
        else:
            pc = None
        
        frame_memory_geometry = None
        if self.use_memory_attention and self.use_plucker:
            frame_memory_geometry = self._build_frame_memory_geometry(
                frame_memory_segments,
                frame_memory_masks,
                frame_memory_pose,
                image_hw,
                x.shape[2],
                x.shape[3],
                x.device,
                x.dtype,
                frame_idx=frame_idx,
            )

        for i, block in enumerate(self.blocks):
            x = block(x, c, current_frame=current_frame, timestep=t, is_last_block= (i+1 == len(self.blocks)), 
                pose_cond=pc, mode=mode, c_action_cond=c_action_cond, reference_length=reference_length,
                frame_memory_segments=frame_memory_segments,
                frame_memory_masks=frame_memory_masks,
                frame_memory_pose=frame_memory_pose,
                image_hw=image_hw,
                frame_memory_geometry=frame_memory_geometry)  # (N, T, H, W, D)
        if frame_memory_segments is not None:
            target_frames = int(frame_memory_segments.get("target", 0))
            x = x[:, :target_frames]
            c = c[:, :target_frames]
            T = target_frames
        x = self.final_layer(x, c) # (N, T, H, W, patch_size ** 2 * out_channels)
        # unpatchify
        x = rearrange(x, "b t h w d -> (b t) h w d")
        x = self.unpatchify(x)  # (N, out_channels, H, W)
        x = rearrange(x, "(b t) c h w -> b t c h w", t=T)
        return x


def DiT_S_2(action_cond_dim, pose_cond_dim, reference_length, 
use_plucker, relative_embedding, 
state_embed_only_on_qk, use_memory_attention, add_timestamp_embedding,
ref_mode, focal_length=0.35, memory_attention_key_only_geometry=True):
    return DiT(
        patch_size=2,
        hidden_size=1024,
        depth=16,
        num_heads=16,
        action_cond_dim=action_cond_dim,
        pose_cond_dim=pose_cond_dim,
        reference_length=reference_length,
        use_plucker=use_plucker,
        relative_embedding=relative_embedding,
        state_embed_only_on_qk=state_embed_only_on_qk,
        use_memory_attention=use_memory_attention,
        add_timestamp_embedding=add_timestamp_embedding,
        memory_attention_key_only_geometry=memory_attention_key_only_geometry,
        ref_mode=ref_mode,
        focal_length=focal_length,
    )


DiT_models = {"DiT-S/2": DiT_S_2}