jhoonjwa commited on
Commit
93d1cba
·
verified ·
1 Parent(s): ffee162

Add slat_dev snapshot 2026-11-11: stage-2 code with test-time soft-mask bias on/off (BIAS_INFERENCE.md) (part 2)

Browse files
Files changed (20) hide show
  1. migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/attention/windowed_attn.py +192 -0
  2. migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/basic.py +569 -0
  3. migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/conv/__init__.py +27 -0
  4. migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/conv/conv_spconv.py +139 -0
  5. migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/linear.py +14 -0
  6. migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/nonlinearity.py +30 -0
  7. migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/norm.py +64 -0
  8. migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/spatial.py +131 -0
  9. migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/transformer/__init__.py +3 -0
  10. migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/transformer/blocks.py +162 -0
  11. migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/transformer/modulated.py +186 -0
  12. migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/spatial.py +65 -0
  13. migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/transformer/__init__.py +3 -0
  14. migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/transformer/blocks.py +197 -0
  15. migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/transformer/modulated.py +341 -0
  16. migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/utils.py +72 -0
  17. migrator/code/slat_dev_20261111/sam3d_objects/model/io.py +210 -0
  18. migrator/code/slat_dev_20261111/sam3d_objects/model/layers/__init__.py +0 -0
  19. migrator/code/slat_dev_20261111/sam3d_objects/model/layers/llama3/__init__.py +0 -0
  20. migrator/code/slat_dev_20261111/sam3d_objects/model/layers/llama3/ff.py +41 -0
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/attention/windowed_attn.py ADDED
@@ -0,0 +1,192 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ from typing import *
3
+ import torch
4
+ import math
5
+ from .. import SparseTensor
6
+ from .. import DEBUG, ATTN
7
+
8
+ if ATTN == "xformers":
9
+ import xformers.ops as xops
10
+ elif ATTN == "flash_attn":
11
+ import flash_attn
12
+ elif ATTN == "sdpa":
13
+ from torch.nn.functional import scaled_dot_product_attention as sdpa
14
+ from .masked_sdpa import masked_sdpa
15
+ else:
16
+ raise ValueError(f"Unknown attention module: {ATTN}")
17
+
18
+
19
+ __all__ = [
20
+ "sparse_windowed_scaled_dot_product_self_attention",
21
+ ]
22
+
23
+
24
+ def calc_window_partition(
25
+ tensor: SparseTensor,
26
+ window_size: Union[int, Tuple[int, ...]],
27
+ shift_window: Union[int, Tuple[int, ...]] = 0,
28
+ ) -> Tuple[torch.Tensor, torch.Tensor, List[int], List[int]]:
29
+ """
30
+ Calculate serialization and partitioning for a set of coordinates.
31
+
32
+ Args:
33
+ tensor (SparseTensor): The input tensor.
34
+ window_size (int): The window size to use.
35
+ shift_window (Tuple[int, ...]): The shift of serialized coordinates.
36
+
37
+ Returns:
38
+ (torch.Tensor): Forwards indices.
39
+ (torch.Tensor): Backwards indices.
40
+ (List[int]): Sequence lengths.
41
+ (List[int]): Sequence batch indices.
42
+ """
43
+ DIM = tensor.coords.shape[1] - 1
44
+ shift_window = (
45
+ (shift_window,) * DIM if isinstance(shift_window, int) else shift_window
46
+ )
47
+ window_size = (window_size,) * DIM if isinstance(window_size, int) else window_size
48
+ shifted_coords = tensor.coords.clone().detach()
49
+ shifted_coords[:, 1:] += torch.tensor(
50
+ shift_window, device=tensor.device, dtype=torch.int32
51
+ ).unsqueeze(0)
52
+
53
+ MAX_COORDS = shifted_coords[:, 1:].max(dim=0).values.tolist()
54
+ NUM_WINDOWS = [math.ceil((mc + 1) / ws) for mc, ws in zip(MAX_COORDS, window_size)]
55
+ OFFSET = torch.cumprod(torch.tensor([1] + NUM_WINDOWS[::-1]), dim=0).tolist()[::-1]
56
+
57
+ shifted_coords[:, 1:] //= torch.tensor(
58
+ window_size, device=tensor.device, dtype=torch.int32
59
+ ).unsqueeze(0)
60
+ shifted_indices = (
61
+ shifted_coords
62
+ * torch.tensor(OFFSET, device=tensor.device, dtype=torch.int32).unsqueeze(0)
63
+ ).sum(dim=1)
64
+ fwd_indices = torch.argsort(shifted_indices)
65
+ bwd_indices = torch.empty_like(fwd_indices)
66
+ bwd_indices[fwd_indices] = torch.arange(fwd_indices.shape[0], device=tensor.device)
67
+ seq_lens = torch.bincount(shifted_indices)
68
+ seq_batch_indices = (
69
+ torch.arange(seq_lens.shape[0], device=tensor.device, dtype=torch.int32)
70
+ // OFFSET[0]
71
+ )
72
+ mask = seq_lens != 0
73
+ seq_lens = seq_lens[mask].tolist()
74
+ seq_batch_indices = seq_batch_indices[mask].tolist()
75
+
76
+ return fwd_indices, bwd_indices, seq_lens, seq_batch_indices
77
+
78
+
79
+ def sparse_windowed_scaled_dot_product_self_attention(
80
+ qkv: SparseTensor, window_size: int, shift_window: Tuple[int, int, int] = (0, 0, 0)
81
+ ) -> SparseTensor:
82
+ """
83
+ Apply windowed scaled dot product self attention to a sparse tensor.
84
+
85
+ Args:
86
+ qkv (SparseTensor): [N, *, 3, H, C] sparse tensor containing Qs, Ks, and Vs.
87
+ window_size (int): The window size to use.
88
+ shift_window (Tuple[int, int, int]): The shift of serialized coordinates.
89
+ shift (int): The shift to use.
90
+ """
91
+ assert (
92
+ len(qkv.shape) == 4 and qkv.shape[1] == 3
93
+ ), f"Invalid shape for qkv, got {qkv.shape}, expected [N, *, 3, H, C]"
94
+
95
+ serialization_spatial_cache_name = f"window_partition_{window_size}_{shift_window}"
96
+ serialization_spatial_cache = qkv.get_spatial_cache(
97
+ serialization_spatial_cache_name
98
+ )
99
+ if serialization_spatial_cache is None:
100
+ fwd_indices, bwd_indices, seq_lens, seq_batch_indices = calc_window_partition(
101
+ qkv, window_size, shift_window
102
+ )
103
+ qkv.register_spatial_cache(
104
+ serialization_spatial_cache_name,
105
+ (fwd_indices, bwd_indices, seq_lens, seq_batch_indices),
106
+ )
107
+ else:
108
+ fwd_indices, bwd_indices, seq_lens, seq_batch_indices = (
109
+ serialization_spatial_cache
110
+ )
111
+
112
+ M = fwd_indices.shape[0]
113
+ T = qkv.feats.shape[0]
114
+ H = qkv.feats.shape[2]
115
+ C = qkv.feats.shape[3]
116
+
117
+ qkv_feats = qkv.feats[fwd_indices] # [M, 3, H, C]
118
+
119
+ if DEBUG:
120
+ start = 0
121
+ qkv_coords = qkv.coords[fwd_indices]
122
+ for i in range(len(seq_lens)):
123
+ seq_coords = qkv_coords[start : start + seq_lens[i]]
124
+ assert (
125
+ seq_coords[:, 0] == seq_batch_indices[i]
126
+ ).all(), (
127
+ f"SparseWindowedScaledDotProductSelfAttention: batch index mismatch"
128
+ )
129
+ assert (
130
+ seq_coords[:, 1:].max(dim=0).values
131
+ - seq_coords[:, 1:].min(dim=0).values
132
+ < window_size
133
+ ).all(), (
134
+ f"SparseWindowedScaledDotProductSelfAttention: window size exceeded"
135
+ )
136
+ start += seq_lens[i]
137
+
138
+ if all([seq_len == window_size for seq_len in seq_lens]):
139
+ B = len(seq_lens)
140
+ N = window_size
141
+ qkv_feats = qkv_feats.reshape(B, N, 3, H, C)
142
+ if ATTN == "xformers":
143
+ q, k, v = qkv_feats.unbind(dim=2) # [B, N, H, C]
144
+ out = xops.memory_efficient_attention(q, k, v) # [B, N, H, C]
145
+ elif ATTN == "flash_attn":
146
+ out = flash_attn.flash_attn_qkvpacked_func(qkv_feats) # [B, N, H, C]
147
+ elif ATTN == "sdpa":
148
+ q, k, v = qkv_feats.unbind(dim=2)
149
+ q = q.permute(0, 2, 1, 3) # [N, H, L, C]
150
+ k = k.permute(0, 2, 1, 3) # [N, H, L, C]
151
+ v = v.permute(0, 2, 1, 3) # [N, H, L, C]
152
+ out = sdpa(q, k, v) # [N, H, L, C]
153
+ out = out.permute(0, 2, 1, 3) # [N, L, H, C]
154
+ else:
155
+ raise ValueError(f"Unknown attention module: {ATTN}")
156
+ out = out.reshape(B * N, H, C) # [M, H, C]
157
+ else:
158
+ if ATTN == "xformers":
159
+ q, k, v = qkv_feats.unbind(dim=1) # [M, H, C]
160
+ q = q.unsqueeze(0) # [1, M, H, C]
161
+ k = k.unsqueeze(0) # [1, M, H, C]
162
+ v = v.unsqueeze(0) # [1, M, H, C]
163
+ mask = xops.fmha.BlockDiagonalMask.from_seqlens(seq_lens)
164
+ out = xops.memory_efficient_attention(q, k, v, mask)[0] # [M, H, C]
165
+ elif ATTN == "flash_attn":
166
+ cu_seqlens = (
167
+ torch.cat(
168
+ [torch.tensor([0]), torch.cumsum(torch.tensor(seq_lens), dim=0)],
169
+ dim=0,
170
+ )
171
+ .to(qkv.device)
172
+ .int()
173
+ )
174
+ out = flash_attn.flash_attn_varlen_qkvpacked_func(
175
+ qkv_feats, cu_seqlens, max(seq_lens)
176
+ ) # [M, H, C]
177
+ elif ATTN == "sdpa":
178
+ q, k, v = qkv_feats.unbind(dim=1)
179
+ q = q.unsqueeze(0)
180
+ k = k.unsqueeze(0)
181
+ v = v.unsqueeze(0)
182
+ out = masked_sdpa(q, k, v, seq_lens, seq_lens)
183
+
184
+ out = out[bwd_indices] # [T, H, C]
185
+
186
+ if DEBUG:
187
+ qkv_coords = qkv_coords[bwd_indices]
188
+ assert torch.equal(
189
+ qkv_coords, qkv.coords
190
+ ), "SparseWindowedScaledDotProductSelfAttention: coordinate mismatch"
191
+
192
+ return qkv.replace(out)
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/basic.py ADDED
@@ -0,0 +1,569 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ from typing import *
3
+ import torch
4
+ from . import BACKEND, DEBUG
5
+
6
+ if BACKEND == "torchsparse":
7
+ from torchsparse import SparseTensor as SparseTensorData
8
+ elif BACKEND == "spconv":
9
+ from spconv.pytorch import SparseConvTensor as SparseTensorData
10
+
11
+ __all__ = [
12
+ "SparseTensor",
13
+ "sparse_batch_broadcast",
14
+ "sparse_batch_op",
15
+ "sparse_cat",
16
+ "sparse_unbind",
17
+ ]
18
+
19
+
20
+ class SparseTensor:
21
+ """
22
+ Sparse tensor with support for both torchsparse and spconv backends.
23
+
24
+ Parameters:
25
+ - feats (torch.Tensor): Features of the sparse tensor.
26
+ - coords (torch.Tensor): Coordinates of the sparse tensor.
27
+ - shape (torch.Size): Shape of the sparse tensor.
28
+ - layout (List[slice]): Layout of the sparse tensor for each batch
29
+ - data (SparseTensorData): Sparse tensor data used for convolusion
30
+
31
+ NOTE:
32
+ - Data corresponding to a same batch should be contiguous.
33
+ - Coords should be in [0, 1023]
34
+ """
35
+
36
+ @overload
37
+ def __init__(
38
+ self,
39
+ feats: torch.Tensor,
40
+ coords: torch.Tensor,
41
+ shape: Optional[torch.Size] = None,
42
+ layout: Optional[List[slice]] = None,
43
+ **kwargs,
44
+ ): ...
45
+
46
+ @overload
47
+ def __init__(
48
+ self,
49
+ data,
50
+ shape: Optional[torch.Size] = None,
51
+ layout: Optional[List[slice]] = None,
52
+ **kwargs,
53
+ ): ...
54
+
55
+ def __init__(self, *args, **kwargs):
56
+ method_id = 0
57
+ if len(args) != 0:
58
+ method_id = 0 if isinstance(args[0], torch.Tensor) else 1
59
+ else:
60
+ method_id = 1 if "data" in kwargs else 0
61
+
62
+ if method_id == 0:
63
+ feats, coords, shape, layout = args + (None,) * (4 - len(args))
64
+ if "feats" in kwargs:
65
+ feats = kwargs["feats"]
66
+ del kwargs["feats"]
67
+ if "coords" in kwargs:
68
+ coords = kwargs["coords"]
69
+ del kwargs["coords"]
70
+ if "shape" in kwargs:
71
+ shape = kwargs["shape"]
72
+ del kwargs["shape"]
73
+ if "layout" in kwargs:
74
+ layout = kwargs["layout"]
75
+ del kwargs["layout"]
76
+
77
+ if shape is None:
78
+ shape = self.__cal_shape(feats, coords)
79
+ if layout is None:
80
+ layout = self.__cal_layout(coords, shape[0])
81
+ if BACKEND == "torchsparse":
82
+ self.data = SparseTensorData(feats, coords, **kwargs)
83
+ elif BACKEND == "spconv":
84
+ spatial_shape = list(coords.max(0)[0] + 1)[1:]
85
+ self.data = SparseTensorData(
86
+ feats.reshape(feats.shape[0], -1),
87
+ coords,
88
+ spatial_shape,
89
+ shape[0],
90
+ **kwargs,
91
+ )
92
+ self.data._features = feats
93
+ elif method_id == 1:
94
+ data, shape, layout = args + (None,) * (3 - len(args))
95
+ if "data" in kwargs:
96
+ data = kwargs["data"]
97
+ del kwargs["data"]
98
+ if "shape" in kwargs:
99
+ shape = kwargs["shape"]
100
+ del kwargs["shape"]
101
+ if "layout" in kwargs:
102
+ layout = kwargs["layout"]
103
+ del kwargs["layout"]
104
+
105
+ self.data = data
106
+ if shape is None:
107
+ shape = self.__cal_shape(self.feats, self.coords)
108
+ if layout is None:
109
+ layout = self.__cal_layout(self.coords, shape[0])
110
+
111
+ self._shape = shape
112
+ self._layout = layout
113
+ self._scale = kwargs.get("scale", (1, 1, 1))
114
+ self._spatial_cache = kwargs.get("spatial_cache", {})
115
+
116
+ if DEBUG:
117
+ try:
118
+ assert (
119
+ self.feats.shape[0] == self.coords.shape[0]
120
+ ), f"Invalid feats shape: {self.feats.shape}, coords shape: {self.coords.shape}"
121
+ assert self.shape == self.__cal_shape(
122
+ self.feats, self.coords
123
+ ), f"Invalid shape: {self.shape}"
124
+ assert self.layout == self.__cal_layout(
125
+ self.coords, self.shape[0]
126
+ ), f"Invalid layout: {self.layout}"
127
+ for i in range(self.shape[0]):
128
+ assert torch.all(
129
+ self.coords[self.layout[i], 0] == i
130
+ ), f"The data of batch {i} is not contiguous"
131
+ except Exception as e:
132
+ print("Debugging information:")
133
+ print(f"- Shape: {self.shape}")
134
+ print(f"- Layout: {self.layout}")
135
+ print(f"- Scale: {self._scale}")
136
+ print(f"- Coords: {self.coords}")
137
+ raise e
138
+
139
+ def __cal_shape(self, feats, coords):
140
+ shape = []
141
+ shape.append(coords[:, 0].max().item() + 1)
142
+ shape.extend([*feats.shape[1:]])
143
+ return torch.Size(shape)
144
+
145
+ def __cal_layout(self, coords, batch_size):
146
+ seq_len = torch.bincount(coords[:, 0], minlength=batch_size)
147
+ offset = torch.cumsum(seq_len, dim=0)
148
+ # one host transfer instead of 2 .item() syncs per batch element (same slices)
149
+ starts, stops = torch.stack([offset - seq_len, offset]).tolist()
150
+ layout = [slice(a, b) for a, b in zip(starts, stops)]
151
+ return layout
152
+
153
+ @property
154
+ def shape(self) -> torch.Size:
155
+ return self._shape
156
+
157
+ def dim(self) -> int:
158
+ return len(self.shape)
159
+
160
+ @property
161
+ def layout(self) -> List[slice]:
162
+ return self._layout
163
+
164
+ @property
165
+ def feats(self) -> torch.Tensor:
166
+ if BACKEND == "torchsparse":
167
+ return self.data.F
168
+ elif BACKEND == "spconv":
169
+ return self.data.features
170
+
171
+ @feats.setter
172
+ def feats(self, value: torch.Tensor):
173
+ if BACKEND == "torchsparse":
174
+ self.data.F = value
175
+ elif BACKEND == "spconv":
176
+ self.data.features = value
177
+
178
+ @property
179
+ def coords(self) -> torch.Tensor:
180
+ if BACKEND == "torchsparse":
181
+ return self.data.C
182
+ elif BACKEND == "spconv":
183
+ return self.data.indices
184
+
185
+ @coords.setter
186
+ def coords(self, value: torch.Tensor):
187
+ if BACKEND == "torchsparse":
188
+ self.data.C = value
189
+ elif BACKEND == "spconv":
190
+ self.data.indices = value
191
+
192
+ @property
193
+ def dtype(self):
194
+ return self.feats.dtype
195
+
196
+ @property
197
+ def device(self):
198
+ return self.feats.device
199
+
200
+ @overload
201
+ def to(self, dtype: torch.dtype) -> "SparseTensor": ...
202
+
203
+ @overload
204
+ def to(
205
+ self,
206
+ device: Optional[Union[str, torch.device]] = None,
207
+ dtype: Optional[torch.dtype] = None,
208
+ ) -> "SparseTensor": ...
209
+
210
+ def to(self, *args, **kwargs) -> "SparseTensor":
211
+ device = None
212
+ dtype = None
213
+ if len(args) == 2:
214
+ device, dtype = args
215
+ elif len(args) == 1:
216
+ if isinstance(args[0], torch.dtype):
217
+ dtype = args[0]
218
+ else:
219
+ device = args[0]
220
+ if "dtype" in kwargs:
221
+ assert dtype is None, "to() received multiple values for argument 'dtype'"
222
+ dtype = kwargs["dtype"]
223
+ if "device" in kwargs:
224
+ assert device is None, "to() received multiple values for argument 'device'"
225
+ device = kwargs["device"]
226
+
227
+ new_feats = self.feats.to(device=device, dtype=dtype)
228
+ new_coords = self.coords.to(device=device)
229
+ return self.replace(new_feats, new_coords)
230
+
231
+ def type(self, dtype):
232
+ new_feats = self.feats.type(dtype)
233
+ return self.replace(new_feats)
234
+
235
+ def cpu(self) -> "SparseTensor":
236
+ new_feats = self.feats.cpu()
237
+ new_coords = self.coords.cpu()
238
+ return self.replace(new_feats, new_coords)
239
+
240
+ def cuda(self) -> "SparseTensor":
241
+ new_feats = self.feats.cuda()
242
+ new_coords = self.coords.cuda()
243
+ return self.replace(new_feats, new_coords)
244
+
245
+ def half(self) -> "SparseTensor":
246
+ new_feats = self.feats.half()
247
+ return self.replace(new_feats)
248
+
249
+ def float(self) -> "SparseTensor":
250
+ new_feats = self.feats.float()
251
+ return self.replace(new_feats)
252
+
253
+ def detach(self) -> "SparseTensor":
254
+ new_coords = self.coords.detach()
255
+ new_feats = self.feats.detach()
256
+ return self.replace(new_feats, new_coords)
257
+
258
+ def dense(self) -> torch.Tensor:
259
+ if BACKEND == "torchsparse":
260
+ return self.data.dense()
261
+ elif BACKEND == "spconv":
262
+ return self.data.dense()
263
+
264
+ def reshape(self, *shape) -> "SparseTensor":
265
+ new_feats = self.feats.reshape(self.feats.shape[0], *shape)
266
+ return self.replace(new_feats)
267
+
268
+ def unbind(self, dim: int) -> List["SparseTensor"]:
269
+ return sparse_unbind(self, dim)
270
+
271
+ def replace(
272
+ self, feats: torch.Tensor, coords: Optional[torch.Tensor] = None
273
+ ) -> "SparseTensor":
274
+ new_shape = [self.shape[0]]
275
+ new_shape.extend(feats.shape[1:])
276
+ if BACKEND == "torchsparse":
277
+ new_data = SparseTensorData(
278
+ feats=feats,
279
+ coords=self.data.coords if coords is None else coords,
280
+ stride=self.data.stride,
281
+ spatial_range=self.data.spatial_range,
282
+ )
283
+ new_data._caches = self.data._caches
284
+ elif BACKEND == "spconv":
285
+ new_data = SparseTensorData(
286
+ self.data.features.reshape(self.data.features.shape[0], -1),
287
+ self.data.indices,
288
+ self.data.spatial_shape,
289
+ self.data.batch_size,
290
+ self.data.grid,
291
+ self.data.voxel_num,
292
+ self.data.indice_dict,
293
+ )
294
+ new_data._features = feats
295
+ new_data.benchmark = self.data.benchmark
296
+ new_data.benchmark_record = self.data.benchmark_record
297
+ new_data.thrust_allocator = self.data.thrust_allocator
298
+ new_data._timer = self.data._timer
299
+ new_data.force_algo = self.data.force_algo
300
+ new_data.int8_scale = self.data.int8_scale
301
+ if coords is not None:
302
+ new_data.indices = coords
303
+ new_tensor = SparseTensor(
304
+ new_data,
305
+ shape=torch.Size(new_shape),
306
+ layout=self.layout,
307
+ scale=self._scale,
308
+ spatial_cache=self._spatial_cache,
309
+ )
310
+ return new_tensor
311
+
312
+ @staticmethod
313
+ def full(aabb, dim, value, dtype=torch.float32, device=None) -> "SparseTensor":
314
+ N, C = dim
315
+ x = torch.arange(aabb[0], aabb[3] + 1)
316
+ y = torch.arange(aabb[1], aabb[4] + 1)
317
+ z = torch.arange(aabb[2], aabb[5] + 1)
318
+ coords = torch.stack(torch.meshgrid(x, y, z, indexing="ij"), dim=-1).reshape(
319
+ -1, 3
320
+ )
321
+ coords = torch.cat(
322
+ [
323
+ torch.arange(N).view(-1, 1).repeat(1, coords.shape[0]).view(-1, 1),
324
+ coords.repeat(N, 1),
325
+ ],
326
+ dim=1,
327
+ ).to(dtype=torch.int32, device=device)
328
+ feats = torch.full((coords.shape[0], C), value, dtype=dtype, device=device)
329
+ return SparseTensor(feats=feats, coords=coords)
330
+
331
+ def __merge_sparse_cache(self, other: "SparseTensor") -> dict:
332
+ new_cache = {}
333
+ for k in set(
334
+ list(self._spatial_cache.keys()) + list(other._spatial_cache.keys())
335
+ ):
336
+ if k in self._spatial_cache:
337
+ new_cache[k] = self._spatial_cache[k]
338
+ if k in other._spatial_cache:
339
+ if k not in new_cache:
340
+ new_cache[k] = other._spatial_cache[k]
341
+ else:
342
+ new_cache[k].update(other._spatial_cache[k])
343
+ return new_cache
344
+
345
+ def __neg__(self) -> "SparseTensor":
346
+ return self.replace(-self.feats)
347
+
348
+ def __elemwise__(
349
+ self, other: Union[torch.Tensor, "SparseTensor"], op: callable
350
+ ) -> "SparseTensor":
351
+ if isinstance(other, torch.Tensor):
352
+ try:
353
+ other = torch.broadcast_to(other, self.shape)
354
+ other = sparse_batch_broadcast(self, other)
355
+ except:
356
+ pass
357
+ if isinstance(other, SparseTensor):
358
+ other = other.feats
359
+ new_feats = op(self.feats, other)
360
+ new_tensor = self.replace(new_feats)
361
+ if isinstance(other, SparseTensor):
362
+ new_tensor._spatial_cache = self.__merge_sparse_cache(other)
363
+ return new_tensor
364
+
365
+ def __add__(
366
+ self, other: Union[torch.Tensor, "SparseTensor", float]
367
+ ) -> "SparseTensor":
368
+ return self.__elemwise__(other, torch.add)
369
+
370
+ def __radd__(
371
+ self, other: Union[torch.Tensor, "SparseTensor", float]
372
+ ) -> "SparseTensor":
373
+ return self.__elemwise__(other, torch.add)
374
+
375
+ def __sub__(
376
+ self, other: Union[torch.Tensor, "SparseTensor", float]
377
+ ) -> "SparseTensor":
378
+ return self.__elemwise__(other, torch.sub)
379
+
380
+ def __rsub__(
381
+ self, other: Union[torch.Tensor, "SparseTensor", float]
382
+ ) -> "SparseTensor":
383
+ return self.__elemwise__(other, lambda x, y: torch.sub(y, x))
384
+
385
+ def __mul__(
386
+ self, other: Union[torch.Tensor, "SparseTensor", float]
387
+ ) -> "SparseTensor":
388
+ return self.__elemwise__(other, torch.mul)
389
+
390
+ def __rmul__(
391
+ self, other: Union[torch.Tensor, "SparseTensor", float]
392
+ ) -> "SparseTensor":
393
+ return self.__elemwise__(other, torch.mul)
394
+
395
+ def __truediv__(
396
+ self, other: Union[torch.Tensor, "SparseTensor", float]
397
+ ) -> "SparseTensor":
398
+ return self.__elemwise__(other, torch.div)
399
+
400
+ def __rtruediv__(
401
+ self, other: Union[torch.Tensor, "SparseTensor", float]
402
+ ) -> "SparseTensor":
403
+ return self.__elemwise__(other, lambda x, y: torch.div(y, x))
404
+
405
+ def __getitem__(self, idx):
406
+ if isinstance(idx, int):
407
+ idx = [idx]
408
+ elif isinstance(idx, slice):
409
+ idx = range(*idx.indices(self.shape[0]))
410
+ elif isinstance(idx, torch.Tensor):
411
+ if idx.dtype == torch.bool:
412
+ assert idx.shape == (
413
+ self.shape[0],
414
+ ), f"Invalid index shape: {idx.shape}"
415
+ idx = idx.nonzero().squeeze(1)
416
+ elif idx.dtype in [torch.int32, torch.int64]:
417
+ assert len(idx.shape) == 1, f"Invalid index shape: {idx.shape}"
418
+ else:
419
+ raise ValueError(f"Unknown index type: {idx.dtype}")
420
+ else:
421
+ raise ValueError(f"Unknown index type: {type(idx)}")
422
+
423
+ coords = []
424
+ feats = []
425
+ for new_idx, old_idx in enumerate(idx):
426
+ coords.append(self.coords[self.layout[old_idx]].clone())
427
+ coords[-1][:, 0] = new_idx
428
+ feats.append(self.feats[self.layout[old_idx]])
429
+ coords = torch.cat(coords, dim=0).contiguous()
430
+ feats = torch.cat(feats, dim=0).contiguous()
431
+ return SparseTensor(feats=feats, coords=coords)
432
+
433
+ def register_spatial_cache(self, key, value) -> None:
434
+ """
435
+ Register a spatial cache.
436
+ The spatial cache can be any thing you want to cache.
437
+ The registery and retrieval of the cache is based on current scale.
438
+ """
439
+ scale_key = str(self._scale)
440
+ if scale_key not in self._spatial_cache:
441
+ self._spatial_cache[scale_key] = {}
442
+ self._spatial_cache[scale_key][key] = value
443
+
444
+ def get_spatial_cache(self, key=None):
445
+ """
446
+ Get a spatial cache.
447
+ """
448
+ scale_key = str(self._scale)
449
+ cur_scale_cache = self._spatial_cache.get(scale_key, {})
450
+ if key is None:
451
+ return cur_scale_cache
452
+ return cur_scale_cache.get(key, None)
453
+
454
+
455
+ class _BatchBroadcast(torch.autograd.Function):
456
+ @staticmethod
457
+ def forward(ctx, other, idx, out_dtype, out_shape):
458
+ ctx.save_for_backward(idx)
459
+ ctx.other_shape = other.shape
460
+ ctx.other_dtype = other.dtype
461
+ rows = other[idx] # [N, *other.shape[1:]]
462
+ # broadcast each row's value like the loop's `broadcasted[rows] = other[k]`
463
+ # (right-aligned): insert singleton dims after the batch dim, then expand
464
+ extra = len(out_shape) - rows.dim()
465
+ if extra > 0:
466
+ rows = rows.reshape(rows.shape[0], *([1] * extra), *rows.shape[1:])
467
+ return rows.to(out_dtype).expand(out_shape).contiguous()
468
+
469
+ @staticmethod
470
+ def backward(ctx, grad):
471
+ (idx,) = ctx.saved_tensors
472
+ acc_dt = torch.float64 if ctx.other_dtype == torch.float64 else torch.float32
473
+ g = grad.to(ctx.other_dtype) # per-row cast (as the loop)
474
+ # sum over any broadcast (non-batch) dims, like ExpandBackward on other[k]
475
+ tgt = ctx.other_shape[1:]
476
+ while g.dim() - 1 > len(tgt):
477
+ g = g.sum(1, dtype=acc_dt).to(ctx.other_dtype)
478
+ for d, sz in enumerate(tgt):
479
+ if sz == 1 and g.shape[d + 1] != 1:
480
+ g = g.sum(d + 1, keepdim=True, dtype=acc_dt).to(ctx.other_dtype)
481
+ acc = torch.zeros((ctx.other_shape[0],) + tuple(g.shape[1:]), dtype=acc_dt,
482
+ device=g.device)
483
+ acc.index_add_(0, idx, g.to(acc_dt))
484
+ return acc.to(ctx.other_dtype), None, None, None
485
+
486
+
487
+ import os as _os
488
+ _BROADCAST_LOOP = _os.environ.get("MVSAM3D_BROADCAST_LOOP", "0") == "1" # 1 = original loop
489
+
490
+
491
+ def sparse_batch_broadcast(input: SparseTensor, other: torch.Tensor) -> torch.Tensor:
492
+ """
493
+ Broadcast a 1D tensor to a sparse tensor along the batch dimension then perform an operation.
494
+
495
+ Args:
496
+ input (torch.Tensor): 1D tensor to broadcast.
497
+ target (SparseTensor): Sparse tensor to broadcast to.
498
+ op (callable): Operation to perform after broadcasting. Defaults to torch.add.
499
+ """
500
+ coords, feats = input.coords, input.feats
501
+ if _BROADCAST_LOOP:
502
+ broadcasted = torch.zeros_like(feats)
503
+ for k in range(input.shape[0]):
504
+ broadcasted[input.layout[k]] = other[k]
505
+ return broadcasted
506
+ # Same forward values as the per-batch slice-assignment loop above, as one
507
+ # gather: row r gets other[batch(r)] cast to feats.dtype. Backward: each
508
+ # row's gradient is cast to other.dtype, then summed per batch element with
509
+ # index_add_ (fp32 accumulation, one final rounding to other.dtype). The
510
+ # summation ORDER differs from the loop's reduction (CUDA index_add_ uses
511
+ # atomics), so the backward matches the loop to fp32 rounding, not bitwise.
512
+ idx = coords[:, 0].long()
513
+ return _BatchBroadcast.apply(other, idx, feats.dtype, tuple(feats.shape))
514
+
515
+
516
+ def sparse_batch_op(
517
+ input: SparseTensor, other: torch.Tensor, op: callable = torch.add
518
+ ) -> SparseTensor:
519
+ """
520
+ Broadcast a 1D tensor to a sparse tensor along the batch dimension then perform an operation.
521
+
522
+ Args:
523
+ input (torch.Tensor): 1D tensor to broadcast.
524
+ target (SparseTensor): Sparse tensor to broadcast to.
525
+ op (callable): Operation to perform after broadcasting. Defaults to torch.add.
526
+ """
527
+ return input.replace(op(input.feats, sparse_batch_broadcast(input, other)))
528
+
529
+
530
+ def sparse_cat(inputs: List[SparseTensor], dim: int = 0) -> SparseTensor:
531
+ """
532
+ Concatenate a list of sparse tensors.
533
+
534
+ Args:
535
+ inputs (List[SparseTensor]): List of sparse tensors to concatenate.
536
+ """
537
+ if dim == 0:
538
+ start = 0
539
+ coords = []
540
+ for input in inputs:
541
+ coords.append(input.coords.clone())
542
+ coords[-1][:, 0] += start
543
+ start += input.shape[0]
544
+ coords = torch.cat(coords, dim=0)
545
+ feats = torch.cat([input.feats for input in inputs], dim=0)
546
+ output = SparseTensor(
547
+ coords=coords,
548
+ feats=feats,
549
+ )
550
+ else:
551
+ feats = torch.cat([input.feats for input in inputs], dim=dim)
552
+ output = inputs[0].replace(feats)
553
+
554
+ return output
555
+
556
+
557
+ def sparse_unbind(input: SparseTensor, dim: int) -> List[SparseTensor]:
558
+ """
559
+ Unbind a sparse tensor along a dimension.
560
+
561
+ Args:
562
+ input (SparseTensor): Sparse tensor to unbind.
563
+ dim (int): Dimension to unbind.
564
+ """
565
+ if dim == 0:
566
+ return [input[i] for i in range(input.shape[0])]
567
+ else:
568
+ feats = input.feats.unbind(dim)
569
+ return [input.replace(f) for f in feats]
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/conv/__init__.py ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ from .. import BACKEND
3
+
4
+
5
+ SPCONV_ALGO = "auto" # 'auto', 'implicit_gemm', 'native'
6
+
7
+
8
+ def __from_env():
9
+ import os
10
+
11
+ global SPCONV_ALGO
12
+ env_spconv_algo = os.environ.get("SPCONV_ALGO")
13
+ if env_spconv_algo is not None and env_spconv_algo in [
14
+ "auto",
15
+ "implicit_gemm",
16
+ "native",
17
+ ]:
18
+ SPCONV_ALGO = env_spconv_algo
19
+ print(f"[SPARSE][CONV] spconv algo: {SPCONV_ALGO}")
20
+
21
+
22
+ __from_env()
23
+
24
+ if BACKEND == "torchsparse":
25
+ from .conv_torchsparse import *
26
+ elif BACKEND == "spconv":
27
+ from .conv_spconv import *
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/conv/conv_spconv.py ADDED
@@ -0,0 +1,139 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ import torch
3
+ import torch.nn as nn
4
+ from .. import SparseTensor
5
+ from .. import DEBUG
6
+ from . import SPCONV_ALGO
7
+
8
+
9
+ class SparseConv3d(nn.Module):
10
+ def __init__(
11
+ self,
12
+ in_channels,
13
+ out_channels,
14
+ kernel_size,
15
+ stride=1,
16
+ dilation=1,
17
+ padding=None,
18
+ bias=True,
19
+ indice_key=None,
20
+ ):
21
+ super(SparseConv3d, self).__init__()
22
+ if "spconv" not in globals():
23
+ import spconv.pytorch as spconv
24
+ algo = None
25
+ if SPCONV_ALGO == "native":
26
+ algo = spconv.ConvAlgo.Native
27
+ elif SPCONV_ALGO == "implicit_gemm":
28
+ algo = spconv.ConvAlgo.MaskImplicitGemm
29
+ if stride == 1 and (padding is None):
30
+ self.conv = spconv.SubMConv3d(
31
+ in_channels,
32
+ out_channels,
33
+ kernel_size,
34
+ dilation=dilation,
35
+ bias=bias,
36
+ indice_key=indice_key,
37
+ algo=algo,
38
+ )
39
+ else:
40
+ self.conv = spconv.SparseConv3d(
41
+ in_channels,
42
+ out_channels,
43
+ kernel_size,
44
+ stride=stride,
45
+ dilation=dilation,
46
+ padding=padding,
47
+ bias=bias,
48
+ indice_key=indice_key,
49
+ algo=algo,
50
+ )
51
+ self.stride = (
52
+ tuple(stride)
53
+ if isinstance(stride, (list, tuple))
54
+ else (stride, stride, stride)
55
+ )
56
+ self.padding = padding
57
+
58
+ def forward(self, x: SparseTensor) -> SparseTensor:
59
+ spatial_changed = any(s != 1 for s in self.stride) or (self.padding is not None)
60
+ new_data = self.conv(x.data)
61
+ new_shape = [x.shape[0], self.conv.out_channels]
62
+ new_layout = None if spatial_changed else x.layout
63
+
64
+ if spatial_changed and (x.shape[0] != 1):
65
+ # spconv was non-1 stride will break the contiguous of the output tensor, sort by the coords
66
+ fwd = new_data.indices[:, 0].argsort()
67
+ bwd = torch.zeros_like(fwd).scatter_(
68
+ 0, fwd, torch.arange(fwd.shape[0], device=fwd.device)
69
+ )
70
+ sorted_feats = new_data.features[fwd]
71
+ sorted_coords = new_data.indices[fwd]
72
+ unsorted_data = new_data
73
+ new_data = spconv.SparseConvTensor(sorted_feats, sorted_coords, unsorted_data.spatial_shape, unsorted_data.batch_size) # type: ignore
74
+
75
+ out = SparseTensor(
76
+ new_data,
77
+ shape=torch.Size(new_shape),
78
+ layout=new_layout,
79
+ scale=tuple([s * stride for s, stride in zip(x._scale, self.stride)]),
80
+ spatial_cache=x._spatial_cache,
81
+ )
82
+
83
+ if spatial_changed and (x.shape[0] != 1):
84
+ out.register_spatial_cache(
85
+ f"conv_{self.stride}_unsorted_data", unsorted_data
86
+ )
87
+ out.register_spatial_cache(f"conv_{self.stride}_sort_bwd", bwd)
88
+
89
+ return out
90
+
91
+
92
+ class SparseInverseConv3d(nn.Module):
93
+ def __init__(
94
+ self,
95
+ in_channels,
96
+ out_channels,
97
+ kernel_size,
98
+ stride=1,
99
+ dilation=1,
100
+ bias=True,
101
+ indice_key=None,
102
+ ):
103
+ super(SparseInverseConv3d, self).__init__()
104
+ if "spconv" not in globals():
105
+ import spconv.pytorch as spconv
106
+ self.conv = spconv.SparseInverseConv3d(
107
+ in_channels, out_channels, kernel_size, bias=bias, indice_key=indice_key
108
+ )
109
+ self.stride = (
110
+ tuple(stride)
111
+ if isinstance(stride, (list, tuple))
112
+ else (stride, stride, stride)
113
+ )
114
+
115
+ def forward(self, x: SparseTensor) -> SparseTensor:
116
+ spatial_changed = any(s != 1 for s in self.stride)
117
+ if spatial_changed:
118
+ # recover the original spconv order
119
+ data = x.get_spatial_cache(f"conv_{self.stride}_unsorted_data")
120
+ bwd = x.get_spatial_cache(f"conv_{self.stride}_sort_bwd")
121
+ data = data.replace_feature(x.feats[bwd])
122
+ if DEBUG:
123
+ assert torch.equal(
124
+ data.indices, x.coords[bwd]
125
+ ), "Recover the original order failed"
126
+ else:
127
+ data = x.data
128
+
129
+ new_data = self.conv(data)
130
+ new_shape = [x.shape[0], self.conv.out_channels]
131
+ new_layout = None if spatial_changed else x.layout
132
+ out = SparseTensor(
133
+ new_data,
134
+ shape=torch.Size(new_shape),
135
+ layout=new_layout,
136
+ scale=tuple([s // stride for s, stride in zip(x._scale, self.stride)]),
137
+ spatial_cache=x._spatial_cache,
138
+ )
139
+ return out
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/linear.py ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ import torch
3
+ import torch.nn as nn
4
+ from . import SparseTensor
5
+
6
+ __all__ = ["SparseLinear"]
7
+
8
+
9
+ class SparseLinear(nn.Linear):
10
+ def __init__(self, in_features, out_features, bias=True):
11
+ super(SparseLinear, self).__init__(in_features, out_features, bias)
12
+
13
+ def forward(self, input: SparseTensor) -> SparseTensor:
14
+ return input.replace(super().forward(input.feats))
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/nonlinearity.py ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ import torch
3
+ import torch.nn as nn
4
+ from . import SparseTensor
5
+
6
+ __all__ = ["SparseReLU", "SparseSiLU", "SparseGELU", "SparseActivation"]
7
+
8
+
9
+ class SparseReLU(nn.ReLU):
10
+ def forward(self, input: SparseTensor) -> SparseTensor:
11
+ return input.replace(super().forward(input.feats))
12
+
13
+
14
+ class SparseSiLU(nn.SiLU):
15
+ def forward(self, input: SparseTensor) -> SparseTensor:
16
+ return input.replace(super().forward(input.feats))
17
+
18
+
19
+ class SparseGELU(nn.GELU):
20
+ def forward(self, input: SparseTensor) -> SparseTensor:
21
+ return input.replace(super().forward(input.feats))
22
+
23
+
24
+ class SparseActivation(nn.Module):
25
+ def __init__(self, activation: nn.Module):
26
+ super().__init__()
27
+ self.activation = activation
28
+
29
+ def forward(self, input: SparseTensor) -> SparseTensor:
30
+ return input.replace(self.activation(input.feats))
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/norm.py ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ import torch
3
+ import torch.nn as nn
4
+ from . import SparseTensor
5
+ from . import DEBUG
6
+
7
+ __all__ = [
8
+ "SparseGroupNorm",
9
+ "SparseLayerNorm",
10
+ "SparseGroupNorm32",
11
+ "SparseLayerNorm32",
12
+ ]
13
+
14
+
15
+ class SparseGroupNorm(nn.GroupNorm):
16
+ def __init__(self, num_groups, num_channels, eps=1e-5, affine=True):
17
+ super(SparseGroupNorm, self).__init__(num_groups, num_channels, eps, affine)
18
+
19
+ def forward(self, input: SparseTensor) -> SparseTensor:
20
+ nfeats = torch.zeros_like(input.feats)
21
+ for k in range(input.shape[0]):
22
+ if DEBUG:
23
+ assert (
24
+ input.coords[input.layout[k], 0] == k
25
+ ).all(), f"SparseGroupNorm: batch index mismatch"
26
+ bfeats = input.feats[input.layout[k]]
27
+ bfeats = bfeats.permute(1, 0).reshape(1, input.shape[1], -1)
28
+ bfeats = super().forward(bfeats)
29
+ bfeats = bfeats.reshape(input.shape[1], -1).permute(1, 0)
30
+ nfeats[input.layout[k]] = bfeats
31
+ return input.replace(nfeats)
32
+
33
+
34
+ class SparseLayerNorm(nn.LayerNorm):
35
+ def __init__(self, normalized_shape, eps=1e-5, elementwise_affine=True):
36
+ super(SparseLayerNorm, self).__init__(normalized_shape, eps, elementwise_affine)
37
+
38
+ def forward(self, input: SparseTensor) -> SparseTensor:
39
+ nfeats = torch.zeros_like(input.feats)
40
+ for k in range(input.shape[0]):
41
+ bfeats = input.feats[input.layout[k]]
42
+ bfeats = bfeats.permute(1, 0).reshape(1, input.shape[1], -1)
43
+ bfeats = super().forward(bfeats)
44
+ bfeats = bfeats.reshape(input.shape[1], -1).permute(1, 0)
45
+ nfeats[input.layout[k]] = bfeats
46
+ return input.replace(nfeats)
47
+
48
+
49
+ class SparseGroupNorm32(SparseGroupNorm):
50
+ """
51
+ A GroupNorm layer that converts to float32 before the forward pass.
52
+ """
53
+
54
+ def forward(self, x: SparseTensor) -> SparseTensor:
55
+ return super().forward(x.float()).type(x.dtype)
56
+
57
+
58
+ class SparseLayerNorm32(SparseLayerNorm):
59
+ """
60
+ A LayerNorm layer that converts to float32 before the forward pass.
61
+ """
62
+
63
+ def forward(self, x: SparseTensor) -> SparseTensor:
64
+ return super().forward(x.float()).type(x.dtype)
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/spatial.py ADDED
@@ -0,0 +1,131 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ from typing import *
3
+ import torch
4
+ import torch.nn as nn
5
+ from . import SparseTensor
6
+
7
+ __all__ = ["SparseDownsample", "SparseUpsample", "SparseSubdivide"]
8
+
9
+
10
+ class SparseDownsample(nn.Module):
11
+ """
12
+ Downsample a sparse tensor by a factor of `factor`.
13
+ Implemented as average pooling.
14
+ """
15
+
16
+ def __init__(self, factor: Union[int, Tuple[int, ...], List[int]]):
17
+ super(SparseDownsample, self).__init__()
18
+ self.factor = tuple(factor) if isinstance(factor, (list, tuple)) else factor
19
+
20
+ def forward(self, input: SparseTensor) -> SparseTensor:
21
+ DIM = input.coords.shape[-1] - 1
22
+ factor = self.factor if isinstance(self.factor, tuple) else (self.factor,) * DIM
23
+ assert DIM == len(
24
+ factor
25
+ ), "Input coordinates must have the same dimension as the downsample factor."
26
+
27
+ coord = list(input.coords.unbind(dim=-1))
28
+ for i, f in enumerate(factor):
29
+ coord[i + 1] = coord[i + 1] // f
30
+
31
+ MAX = [coord[i + 1].max().item() + 1 for i in range(DIM)]
32
+ OFFSET = torch.cumprod(torch.tensor(MAX[::-1]), 0).tolist()[::-1] + [1]
33
+ code = sum([c * o for c, o in zip(coord, OFFSET)])
34
+ code, idx = code.unique(return_inverse=True)
35
+
36
+ new_feats = torch.scatter_reduce(
37
+ torch.zeros(
38
+ code.shape[0],
39
+ input.feats.shape[1],
40
+ device=input.feats.device,
41
+ dtype=input.feats.dtype,
42
+ ),
43
+ dim=0,
44
+ index=idx.unsqueeze(1).expand(-1, input.feats.shape[1]),
45
+ src=input.feats,
46
+ reduce="mean",
47
+ )
48
+ new_coords = torch.stack(
49
+ [code // OFFSET[0]]
50
+ + [(code // OFFSET[i + 1]) % MAX[i] for i in range(DIM)],
51
+ dim=-1,
52
+ )
53
+ out = SparseTensor(
54
+ new_feats,
55
+ new_coords,
56
+ input.shape,
57
+ )
58
+ out._scale = tuple([s // f for s, f in zip(input._scale, factor)])
59
+ out._spatial_cache = input._spatial_cache
60
+
61
+ out.register_spatial_cache(f"upsample_{factor}_coords", input.coords)
62
+ out.register_spatial_cache(f"upsample_{factor}_layout", input.layout)
63
+ out.register_spatial_cache(f"upsample_{factor}_idx", idx)
64
+
65
+ return out
66
+
67
+
68
+ class SparseUpsample(nn.Module):
69
+ """
70
+ Upsample a sparse tensor by a factor of `factor`.
71
+ Implemented as nearest neighbor interpolation.
72
+ """
73
+
74
+ def __init__(self, factor: Union[int, Tuple[int, int, int], List[int]]):
75
+ super(SparseUpsample, self).__init__()
76
+ self.factor = tuple(factor) if isinstance(factor, (list, tuple)) else factor
77
+
78
+ def forward(self, input: SparseTensor) -> SparseTensor:
79
+ DIM = input.coords.shape[-1] - 1
80
+ factor = self.factor if isinstance(self.factor, tuple) else (self.factor,) * DIM
81
+ assert DIM == len(
82
+ factor
83
+ ), "Input coordinates must have the same dimension as the upsample factor."
84
+
85
+ new_coords = input.get_spatial_cache(f"upsample_{factor}_coords")
86
+ new_layout = input.get_spatial_cache(f"upsample_{factor}_layout")
87
+ idx = input.get_spatial_cache(f"upsample_{factor}_idx")
88
+ if any([x is None for x in [new_coords, new_layout, idx]]):
89
+ raise ValueError(
90
+ "Upsample cache not found. SparseUpsample must be paired with SparseDownsample."
91
+ )
92
+ new_feats = input.feats[idx]
93
+ out = SparseTensor(new_feats, new_coords, input.shape, new_layout)
94
+ out._scale = tuple([s * f for s, f in zip(input._scale, factor)])
95
+ out._spatial_cache = input._spatial_cache
96
+ return out
97
+
98
+
99
+ class SparseSubdivide(nn.Module):
100
+ """
101
+ Upsample a sparse tensor by a factor of `factor`.
102
+ Implemented as nearest neighbor interpolation.
103
+ """
104
+
105
+ def __init__(self):
106
+ super(SparseSubdivide, self).__init__()
107
+
108
+ def forward(self, input: SparseTensor) -> SparseTensor:
109
+ DIM = input.coords.shape[-1] - 1
110
+ # upsample scale=2^DIM
111
+ n_cube = torch.ones([2] * DIM, device=input.device, dtype=torch.int)
112
+ n_coords = torch.nonzero(n_cube)
113
+ n_coords = torch.cat([torch.zeros_like(n_coords[:, :1]), n_coords], dim=-1)
114
+ factor = n_coords.shape[0]
115
+ assert factor == 2**DIM
116
+ # print(n_coords.shape)
117
+ new_coords = input.coords.clone()
118
+ new_coords[:, 1:] *= 2
119
+ new_coords = new_coords.unsqueeze(1) + n_coords.unsqueeze(0).to(
120
+ new_coords.dtype
121
+ )
122
+
123
+ new_feats = input.feats.unsqueeze(1).expand(
124
+ input.feats.shape[0], factor, *input.feats.shape[1:]
125
+ )
126
+ out = SparseTensor(
127
+ new_feats.flatten(0, 1), new_coords.flatten(0, 1), input.shape
128
+ )
129
+ out._scale = input._scale * 2
130
+ out._spatial_cache = input._spatial_cache
131
+ return out
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/transformer/__init__.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ from .blocks import *
3
+ from .modulated import *
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/transformer/blocks.py ADDED
@@ -0,0 +1,162 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ from typing import *
3
+ import torch
4
+ import torch.nn as nn
5
+ from ..basic import SparseTensor
6
+ from ..linear import SparseLinear
7
+ from ..nonlinearity import SparseGELU
8
+ from ..attention import SparseMultiHeadAttention, SerializeMode
9
+ from ...norm import LayerNorm32
10
+
11
+
12
+ class SparseFeedForwardNet(nn.Module):
13
+ def __init__(self, channels: int, mlp_ratio: float = 4.0):
14
+ super().__init__()
15
+ self.mlp = nn.Sequential(
16
+ SparseLinear(channels, int(channels * mlp_ratio)),
17
+ SparseGELU(approximate="tanh"),
18
+ SparseLinear(int(channels * mlp_ratio), channels),
19
+ )
20
+
21
+ def forward(self, x: SparseTensor) -> SparseTensor:
22
+ return self.mlp(x)
23
+
24
+
25
+ class SparseTransformerBlock(nn.Module):
26
+ """
27
+ Sparse Transformer block (MSA + FFN).
28
+ """
29
+
30
+ def __init__(
31
+ self,
32
+ channels: int,
33
+ num_heads: int,
34
+ mlp_ratio: float = 4.0,
35
+ attn_mode: Literal[
36
+ "full", "shift_window", "shift_sequence", "shift_order", "swin"
37
+ ] = "full",
38
+ window_size: Optional[int] = None,
39
+ shift_sequence: Optional[int] = None,
40
+ shift_window: Optional[Tuple[int, int, int]] = None,
41
+ serialize_mode: Optional[SerializeMode] = None,
42
+ use_checkpoint: bool = False,
43
+ use_rope: bool = False,
44
+ qk_rms_norm: bool = False,
45
+ qkv_bias: bool = True,
46
+ ln_affine: bool = False,
47
+ ):
48
+ super().__init__()
49
+ self.use_checkpoint = use_checkpoint
50
+ self.norm1 = LayerNorm32(channels, elementwise_affine=ln_affine, eps=1e-6)
51
+ self.norm2 = LayerNorm32(channels, elementwise_affine=ln_affine, eps=1e-6)
52
+ self.attn = SparseMultiHeadAttention(
53
+ channels,
54
+ num_heads=num_heads,
55
+ attn_mode=attn_mode,
56
+ window_size=window_size,
57
+ shift_sequence=shift_sequence,
58
+ shift_window=shift_window,
59
+ serialize_mode=serialize_mode,
60
+ qkv_bias=qkv_bias,
61
+ use_rope=use_rope,
62
+ qk_rms_norm=qk_rms_norm,
63
+ )
64
+ self.mlp = SparseFeedForwardNet(
65
+ channels,
66
+ mlp_ratio=mlp_ratio,
67
+ )
68
+
69
+ def _forward(self, x: SparseTensor) -> SparseTensor:
70
+ h = x.replace(self.norm1(x.feats))
71
+ h = self.attn(h)
72
+ x = x + h
73
+ h = x.replace(self.norm2(x.feats))
74
+ h = self.mlp(h)
75
+ x = x + h
76
+ return x
77
+
78
+ def forward(self, x: SparseTensor) -> SparseTensor:
79
+ if self.use_checkpoint:
80
+ return torch.utils.checkpoint.checkpoint(
81
+ self._forward, x, use_reentrant=False
82
+ )
83
+ else:
84
+ return self._forward(x)
85
+
86
+
87
+ class SparseTransformerCrossBlock(nn.Module):
88
+ """
89
+ Sparse Transformer cross-attention block (MSA + MCA + FFN).
90
+ """
91
+
92
+ def __init__(
93
+ self,
94
+ channels: int,
95
+ ctx_channels: int,
96
+ num_heads: int,
97
+ mlp_ratio: float = 4.0,
98
+ attn_mode: Literal[
99
+ "full", "shift_window", "shift_sequence", "shift_order", "swin"
100
+ ] = "full",
101
+ window_size: Optional[int] = None,
102
+ shift_sequence: Optional[int] = None,
103
+ shift_window: Optional[Tuple[int, int, int]] = None,
104
+ serialize_mode: Optional[SerializeMode] = None,
105
+ use_checkpoint: bool = False,
106
+ use_rope: bool = False,
107
+ qk_rms_norm: bool = False,
108
+ qk_rms_norm_cross: bool = False,
109
+ qkv_bias: bool = True,
110
+ ln_affine: bool = False,
111
+ ):
112
+ super().__init__()
113
+ self.use_checkpoint = use_checkpoint
114
+ self.norm1 = LayerNorm32(channels, elementwise_affine=ln_affine, eps=1e-6)
115
+ self.norm2 = LayerNorm32(channels, elementwise_affine=ln_affine, eps=1e-6)
116
+ self.norm3 = LayerNorm32(channels, elementwise_affine=ln_affine, eps=1e-6)
117
+ self.self_attn = SparseMultiHeadAttention(
118
+ channels,
119
+ num_heads=num_heads,
120
+ type="self",
121
+ attn_mode=attn_mode,
122
+ window_size=window_size,
123
+ shift_sequence=shift_sequence,
124
+ shift_window=shift_window,
125
+ serialize_mode=serialize_mode,
126
+ qkv_bias=qkv_bias,
127
+ use_rope=use_rope,
128
+ qk_rms_norm=qk_rms_norm,
129
+ )
130
+ self.cross_attn = SparseMultiHeadAttention(
131
+ channels,
132
+ ctx_channels=ctx_channels,
133
+ num_heads=num_heads,
134
+ type="cross",
135
+ attn_mode="full",
136
+ qkv_bias=qkv_bias,
137
+ qk_rms_norm=qk_rms_norm_cross,
138
+ )
139
+ self.mlp = SparseFeedForwardNet(
140
+ channels,
141
+ mlp_ratio=mlp_ratio,
142
+ )
143
+
144
+ def _forward(self, x: SparseTensor, mod: torch.Tensor, context: torch.Tensor):
145
+ h = x.replace(self.norm1(x.feats))
146
+ h = self.self_attn(h)
147
+ x = x + h
148
+ h = x.replace(self.norm2(x.feats))
149
+ h = self.cross_attn(h, context)
150
+ x = x + h
151
+ h = x.replace(self.norm3(x.feats))
152
+ h = self.mlp(h)
153
+ x = x + h
154
+ return x
155
+
156
+ def forward(self, x: SparseTensor, context: torch.Tensor):
157
+ if self.use_checkpoint:
158
+ return torch.utils.checkpoint.checkpoint(
159
+ self._forward, x, context, use_reentrant=False
160
+ )
161
+ else:
162
+ return self._forward(x, context)
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/transformer/modulated.py ADDED
@@ -0,0 +1,186 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ from typing import *
3
+ import torch
4
+ import torch.nn as nn
5
+ from ..basic import SparseTensor
6
+ from ..attention import SparseMultiHeadAttention, SerializeMode
7
+ from ...norm import LayerNorm32
8
+ from .blocks import SparseFeedForwardNet
9
+
10
+
11
+ class ModulatedSparseTransformerBlock(nn.Module):
12
+ """
13
+ Sparse Transformer block (MSA + FFN) with adaptive layer norm conditioning.
14
+ """
15
+
16
+ def __init__(
17
+ self,
18
+ channels: int,
19
+ num_heads: int,
20
+ mlp_ratio: float = 4.0,
21
+ attn_mode: Literal[
22
+ "full", "shift_window", "shift_sequence", "shift_order", "swin"
23
+ ] = "full",
24
+ window_size: Optional[int] = None,
25
+ shift_sequence: Optional[int] = None,
26
+ shift_window: Optional[Tuple[int, int, int]] = None,
27
+ serialize_mode: Optional[SerializeMode] = None,
28
+ use_checkpoint: bool = False,
29
+ use_rope: bool = False,
30
+ qk_rms_norm: bool = False,
31
+ qkv_bias: bool = True,
32
+ share_mod: bool = False,
33
+ ):
34
+ super().__init__()
35
+ self.use_checkpoint = use_checkpoint
36
+ self.share_mod = share_mod
37
+ self.norm1 = LayerNorm32(channels, elementwise_affine=False, eps=1e-6)
38
+ self.norm2 = LayerNorm32(channels, elementwise_affine=False, eps=1e-6)
39
+ self.attn = SparseMultiHeadAttention(
40
+ channels,
41
+ num_heads=num_heads,
42
+ attn_mode=attn_mode,
43
+ window_size=window_size,
44
+ shift_sequence=shift_sequence,
45
+ shift_window=shift_window,
46
+ serialize_mode=serialize_mode,
47
+ qkv_bias=qkv_bias,
48
+ use_rope=use_rope,
49
+ qk_rms_norm=qk_rms_norm,
50
+ )
51
+ self.mlp = SparseFeedForwardNet(
52
+ channels,
53
+ mlp_ratio=mlp_ratio,
54
+ )
55
+ if not share_mod:
56
+ self.adaLN_modulation = nn.Sequential(
57
+ nn.SiLU(), nn.Linear(channels, 6 * channels, bias=True)
58
+ )
59
+
60
+ def _forward(self, x: SparseTensor, mod: torch.Tensor) -> SparseTensor:
61
+ if self.share_mod:
62
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mod.chunk(
63
+ 6, dim=1
64
+ )
65
+ else:
66
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
67
+ self.adaLN_modulation(mod).chunk(6, dim=1)
68
+ )
69
+ h = x.replace(self.norm1(x.feats))
70
+ h = h * (1 + scale_msa) + shift_msa
71
+ h = self.attn(h)
72
+ h = h * gate_msa
73
+ x = x + h
74
+ h = x.replace(self.norm2(x.feats))
75
+ h = h * (1 + scale_mlp) + shift_mlp
76
+ h = self.mlp(h)
77
+ h = h * gate_mlp
78
+ x = x + h
79
+ return x
80
+
81
+ def forward(self, x: SparseTensor, mod: torch.Tensor) -> SparseTensor:
82
+ if self.use_checkpoint:
83
+ return torch.utils.checkpoint.checkpoint(
84
+ self._forward, x, mod, use_reentrant=False
85
+ )
86
+ else:
87
+ return self._forward(x, mod)
88
+
89
+
90
+ class ModulatedSparseTransformerCrossBlock(nn.Module):
91
+ """
92
+ Sparse Transformer cross-attention block (MSA + MCA + FFN) with adaptive layer norm conditioning.
93
+ """
94
+
95
+ def __init__(
96
+ self,
97
+ channels: int,
98
+ ctx_channels: int,
99
+ num_heads: int,
100
+ mlp_ratio: float = 4.0,
101
+ attn_mode: Literal[
102
+ "full", "shift_window", "shift_sequence", "shift_order", "swin"
103
+ ] = "full",
104
+ window_size: Optional[int] = None,
105
+ shift_sequence: Optional[int] = None,
106
+ shift_window: Optional[Tuple[int, int, int]] = None,
107
+ serialize_mode: Optional[SerializeMode] = None,
108
+ use_checkpoint: bool = False,
109
+ use_rope: bool = False,
110
+ qk_rms_norm: bool = False,
111
+ qk_rms_norm_cross: bool = False,
112
+ qkv_bias: bool = True,
113
+ share_mod: bool = False,
114
+ ):
115
+ super().__init__()
116
+ self.use_checkpoint = use_checkpoint
117
+ self.share_mod = share_mod
118
+ self.norm1 = LayerNorm32(channels, elementwise_affine=False, eps=1e-6)
119
+ self.norm2 = LayerNorm32(channels, elementwise_affine=True, eps=1e-6)
120
+ self.norm3 = LayerNorm32(channels, elementwise_affine=False, eps=1e-6)
121
+ self.self_attn = SparseMultiHeadAttention(
122
+ channels,
123
+ num_heads=num_heads,
124
+ type="self",
125
+ attn_mode=attn_mode,
126
+ window_size=window_size,
127
+ shift_sequence=shift_sequence,
128
+ shift_window=shift_window,
129
+ serialize_mode=serialize_mode,
130
+ qkv_bias=qkv_bias,
131
+ use_rope=use_rope,
132
+ qk_rms_norm=qk_rms_norm,
133
+ )
134
+ self.cross_attn = SparseMultiHeadAttention(
135
+ channels,
136
+ ctx_channels=ctx_channels,
137
+ num_heads=num_heads,
138
+ type="cross",
139
+ attn_mode="full",
140
+ qkv_bias=qkv_bias,
141
+ qk_rms_norm=qk_rms_norm_cross,
142
+ )
143
+ self.mlp = SparseFeedForwardNet(
144
+ channels,
145
+ mlp_ratio=mlp_ratio,
146
+ )
147
+ if not share_mod:
148
+ self.adaLN_modulation = nn.Sequential(
149
+ nn.SiLU(), nn.Linear(channels, 6 * channels, bias=True)
150
+ )
151
+
152
+ def _forward(
153
+ self, x: SparseTensor, mod: torch.Tensor, context: torch.Tensor
154
+ ) -> SparseTensor:
155
+ if self.share_mod:
156
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mod.chunk(
157
+ 6, dim=1
158
+ )
159
+ else:
160
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
161
+ self.adaLN_modulation(mod).chunk(6, dim=1)
162
+ )
163
+ h = x.replace(self.norm1(x.feats))
164
+ h = h * (1 + scale_msa) + shift_msa
165
+ h = self.self_attn(h)
166
+ h = h * gate_msa
167
+ x = x + h
168
+ h = x.replace(self.norm2(x.feats))
169
+ h = self.cross_attn(h, context)
170
+ x = x + h
171
+ h = x.replace(self.norm3(x.feats))
172
+ h = h * (1 + scale_mlp) + shift_mlp
173
+ h = self.mlp(h)
174
+ h = h * gate_mlp
175
+ x = x + h
176
+ return x
177
+
178
+ def forward(
179
+ self, x: SparseTensor, mod: torch.Tensor, context: torch.Tensor
180
+ ) -> SparseTensor:
181
+ if self.use_checkpoint:
182
+ return torch.utils.checkpoint.checkpoint(
183
+ self._forward, x, mod, context, use_reentrant=False
184
+ )
185
+ else:
186
+ return self._forward(x, mod, context)
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/spatial.py ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ import torch
3
+
4
+
5
+ def pixel_shuffle_3d(x: torch.Tensor, scale_factor: int) -> torch.Tensor:
6
+ """
7
+ 3D pixel shuffle.
8
+ """
9
+ B, C, H, W, D = x.shape
10
+ C_ = C // scale_factor**3
11
+ x = x.reshape(B, C_, scale_factor, scale_factor, scale_factor, H, W, D)
12
+ x = x.permute(0, 1, 5, 2, 6, 3, 7, 4)
13
+ x = x.reshape(B, C_, H * scale_factor, W * scale_factor, D * scale_factor)
14
+ return x
15
+
16
+
17
+ def patchify(x: torch.Tensor, patch_size: int):
18
+ """
19
+ Patchify a tensor.
20
+
21
+ Args:
22
+ x (torch.Tensor): (N, C, *spatial) tensor
23
+ patch_size (int): Patch size
24
+ """
25
+ DIM = x.dim() - 2
26
+ for d in range(2, DIM + 2):
27
+ assert (
28
+ x.shape[d] % patch_size == 0
29
+ ), f"Dimension {d} of input tensor must be divisible by patch size, got {x.shape[d]} and {patch_size}"
30
+
31
+ x = x.reshape(
32
+ *x.shape[:2],
33
+ *sum([[x.shape[d] // patch_size, patch_size] for d in range(2, DIM + 2)], []),
34
+ )
35
+ x = x.permute(
36
+ 0, 1, *([2 * i + 3 for i in range(DIM)] + [2 * i + 2 for i in range(DIM)])
37
+ )
38
+ x = x.reshape(x.shape[0], x.shape[1] * (patch_size**DIM), *(x.shape[-DIM:]))
39
+ return x
40
+
41
+
42
+ def unpatchify(x: torch.Tensor, patch_size: int):
43
+ """
44
+ Unpatchify a tensor.
45
+
46
+ Args:
47
+ x (torch.Tensor): (N, C, *spatial) tensor
48
+ patch_size (int): Patch size
49
+ """
50
+ DIM = x.dim() - 2
51
+ assert (
52
+ x.shape[1] % (patch_size**DIM) == 0
53
+ ), f"Second dimension of input tensor must be divisible by patch size to unpatchify, got {x.shape[1]} and {patch_size ** DIM}"
54
+
55
+ x = x.reshape(
56
+ x.shape[0],
57
+ x.shape[1] // (patch_size**DIM),
58
+ *([patch_size] * DIM),
59
+ *(x.shape[-DIM:]),
60
+ )
61
+ x = x.permute(0, 1, *(sum([[2 + DIM + i, 2 + i] for i in range(DIM)], [])))
62
+ x = x.reshape(
63
+ x.shape[0], x.shape[1], *[x.shape[2 + 2 * i] * patch_size for i in range(DIM)]
64
+ )
65
+ return x
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/transformer/__init__.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ from .blocks import *
3
+ from .modulated import *
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/transformer/blocks.py ADDED
@@ -0,0 +1,197 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ from typing import *
3
+ import torch
4
+ import torch.nn as nn
5
+ from ..attention import MultiHeadAttention
6
+ from ..norm import LayerNorm32
7
+
8
+
9
+ class AbsolutePositionEmbedder(nn.Module):
10
+ """
11
+ Embeds spatial positions into vector representations.
12
+ """
13
+
14
+ def __init__(self, channels: int, in_channels: int = 3):
15
+ super().__init__()
16
+ self.channels = channels
17
+ self.in_channels = in_channels
18
+ self.freq_dim = channels // in_channels // 2
19
+ self.freqs = torch.arange(self.freq_dim, dtype=torch.float32) / self.freq_dim
20
+ self.freqs = 1.0 / (10000**self.freqs)
21
+
22
+ def _sin_cos_embedding(self, x: torch.Tensor) -> torch.Tensor:
23
+ """
24
+ Create sinusoidal position embeddings.
25
+
26
+ Args:
27
+ x: a 1-D Tensor of N indices
28
+
29
+ Returns:
30
+ an (N, D) Tensor of positional embeddings.
31
+ """
32
+ self.freqs = self.freqs.to(x.device)
33
+ out = torch.outer(x, self.freqs)
34
+ out = torch.cat([torch.sin(out), torch.cos(out)], dim=-1)
35
+ return out
36
+
37
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
38
+ """
39
+ Args:
40
+ x (torch.Tensor): (N, D) tensor of spatial positions
41
+ """
42
+ N, D = x.shape
43
+ assert (
44
+ D == self.in_channels
45
+ ), "Input dimension must match number of input channels"
46
+ embed = self._sin_cos_embedding(x.reshape(-1))
47
+ embed = embed.reshape(N, -1)
48
+ if embed.shape[1] < self.channels:
49
+ embed = torch.cat(
50
+ [
51
+ embed,
52
+ torch.zeros(N, self.channels - embed.shape[1], device=embed.device),
53
+ ],
54
+ dim=-1,
55
+ )
56
+ return embed
57
+
58
+
59
+ class FeedForwardNet(nn.Module):
60
+ def __init__(self, channels: int, mlp_ratio: float = 4.0):
61
+ super().__init__()
62
+ self.mlp = nn.Sequential(
63
+ nn.Linear(channels, int(channels * mlp_ratio)),
64
+ nn.GELU(approximate="tanh"),
65
+ nn.Linear(int(channels * mlp_ratio), channels),
66
+ )
67
+
68
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
69
+ return self.mlp(x)
70
+
71
+
72
+ class TransformerBlock(nn.Module):
73
+ """
74
+ Transformer block (MSA + FFN).
75
+ """
76
+
77
+ def __init__(
78
+ self,
79
+ channels: int,
80
+ num_heads: int,
81
+ mlp_ratio: float = 4.0,
82
+ attn_mode: Literal["full", "windowed"] = "full",
83
+ window_size: Optional[int] = None,
84
+ shift_window: Optional[int] = None,
85
+ use_checkpoint: bool = False,
86
+ use_rope: bool = False,
87
+ qk_rms_norm: bool = False,
88
+ qkv_bias: bool = True,
89
+ ln_affine: bool = False,
90
+ ):
91
+ super().__init__()
92
+ self.use_checkpoint = use_checkpoint
93
+ self.norm1 = LayerNorm32(channels, elementwise_affine=ln_affine, eps=1e-6)
94
+ self.norm2 = LayerNorm32(channels, elementwise_affine=ln_affine, eps=1e-6)
95
+ self.attn = MultiHeadAttention(
96
+ channels,
97
+ num_heads=num_heads,
98
+ attn_mode=attn_mode,
99
+ window_size=window_size,
100
+ shift_window=shift_window,
101
+ qkv_bias=qkv_bias,
102
+ use_rope=use_rope,
103
+ qk_rms_norm=qk_rms_norm,
104
+ )
105
+ self.mlp = FeedForwardNet(
106
+ channels,
107
+ mlp_ratio=mlp_ratio,
108
+ )
109
+
110
+ def _forward(self, x: torch.Tensor) -> torch.Tensor:
111
+ h = self.norm1(x)
112
+ h = self.attn(h)
113
+ x = x + h
114
+ h = self.norm2(x)
115
+ h = self.mlp(h)
116
+ x = x + h
117
+ return x
118
+
119
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
120
+ if self.use_checkpoint:
121
+ return torch.utils.checkpoint.checkpoint(
122
+ self._forward, x, use_reentrant=False
123
+ )
124
+ else:
125
+ return self._forward(x)
126
+
127
+
128
+ class TransformerCrossBlock(nn.Module):
129
+ """
130
+ Transformer cross-attention block (MSA + MCA + FFN).
131
+ """
132
+
133
+ def __init__(
134
+ self,
135
+ channels: int,
136
+ ctx_channels: int,
137
+ num_heads: int,
138
+ mlp_ratio: float = 4.0,
139
+ attn_mode: Literal["full", "windowed"] = "full",
140
+ window_size: Optional[int] = None,
141
+ shift_window: Optional[Tuple[int, int, int]] = None,
142
+ use_checkpoint: bool = False,
143
+ use_rope: bool = False,
144
+ qk_rms_norm: bool = False,
145
+ qk_rms_norm_cross: bool = False,
146
+ qkv_bias: bool = True,
147
+ ln_affine: bool = False,
148
+ ):
149
+ super().__init__()
150
+ self.use_checkpoint = use_checkpoint
151
+ self.norm1 = LayerNorm32(channels, elementwise_affine=ln_affine, eps=1e-6)
152
+ self.norm2 = LayerNorm32(channels, elementwise_affine=ln_affine, eps=1e-6)
153
+ self.norm3 = LayerNorm32(channels, elementwise_affine=ln_affine, eps=1e-6)
154
+ self.self_attn = MultiHeadAttention(
155
+ channels,
156
+ num_heads=num_heads,
157
+ type="self",
158
+ attn_mode=attn_mode,
159
+ window_size=window_size,
160
+ shift_window=shift_window,
161
+ qkv_bias=qkv_bias,
162
+ use_rope=use_rope,
163
+ qk_rms_norm=qk_rms_norm,
164
+ )
165
+ self.cross_attn = MultiHeadAttention(
166
+ channels,
167
+ ctx_channels=ctx_channels,
168
+ num_heads=num_heads,
169
+ type="cross",
170
+ attn_mode="full",
171
+ qkv_bias=qkv_bias,
172
+ qk_rms_norm=qk_rms_norm_cross,
173
+ )
174
+ self.mlp = FeedForwardNet(
175
+ channels,
176
+ mlp_ratio=mlp_ratio,
177
+ )
178
+
179
+ def _forward(self, x: torch.Tensor, context: torch.Tensor):
180
+ h = self.norm1(x)
181
+ h = self.self_attn(h)
182
+ x = x + h
183
+ h = self.norm2(x)
184
+ h = self.cross_attn(h, context)
185
+ x = x + h
186
+ h = self.norm3(x)
187
+ h = self.mlp(h)
188
+ x = x + h
189
+ return x
190
+
191
+ def forward(self, x: torch.Tensor, context: torch.Tensor):
192
+ if self.use_checkpoint:
193
+ return torch.utils.checkpoint.checkpoint(
194
+ self._forward, x, context, use_reentrant=False
195
+ )
196
+ else:
197
+ return self._forward(x, context)
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/transformer/modulated.py ADDED
@@ -0,0 +1,341 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ from functools import partial
3
+ from typing import *
4
+ from torch.utils import _pytree
5
+ import torch
6
+ import torch.nn as nn
7
+ from ..attention import MultiHeadAttention, MOTMultiHeadSelfAttention
8
+ from ..norm import LayerNorm32
9
+ from .blocks import FeedForwardNet
10
+
11
+
12
+ class ModulatedTransformerBlock(nn.Module):
13
+ """
14
+ Transformer block (MSA + FFN) with adaptive layer norm conditioning.
15
+ """
16
+
17
+ def __init__(
18
+ self,
19
+ channels: int,
20
+ num_heads: int,
21
+ mlp_ratio: float = 4.0,
22
+ attn_mode: Literal["full", "windowed"] = "full",
23
+ window_size: Optional[int] = None,
24
+ shift_window: Optional[Tuple[int, int, int]] = None,
25
+ use_checkpoint: bool = False,
26
+ use_rope: bool = False,
27
+ qk_rms_norm: bool = False,
28
+ qkv_bias: bool = True,
29
+ share_mod: bool = False,
30
+ ):
31
+ super().__init__()
32
+ self.use_checkpoint = use_checkpoint
33
+ self.share_mod = share_mod
34
+ self.norm1 = LayerNorm32(channels, elementwise_affine=False, eps=1e-6)
35
+ self.norm2 = LayerNorm32(channels, elementwise_affine=False, eps=1e-6)
36
+ self.attn = MultiHeadAttention(
37
+ channels,
38
+ num_heads=num_heads,
39
+ attn_mode=attn_mode,
40
+ window_size=window_size,
41
+ shift_window=shift_window,
42
+ qkv_bias=qkv_bias,
43
+ use_rope=use_rope,
44
+ qk_rms_norm=qk_rms_norm,
45
+ )
46
+ self.mlp = FeedForwardNet(
47
+ channels,
48
+ mlp_ratio=mlp_ratio,
49
+ )
50
+ if not share_mod:
51
+ self.adaLN_modulation = nn.Sequential(
52
+ nn.SiLU(), nn.Linear(channels, 6 * channels, bias=True)
53
+ )
54
+
55
+ def _forward(self, x: torch.Tensor, mod: torch.Tensor) -> torch.Tensor:
56
+ if self.share_mod:
57
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mod.chunk(
58
+ 6, dim=1
59
+ )
60
+ else:
61
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
62
+ self.adaLN_modulation(mod).chunk(6, dim=1)
63
+ )
64
+ h = self.norm1(x)
65
+ h = h * (1 + scale_msa.unsqueeze(1)) + shift_msa.unsqueeze(1)
66
+ h = self.attn(h)
67
+ h = h * gate_msa.unsqueeze(1)
68
+ x = x + h
69
+ h = self.norm2(x)
70
+ h = h * (1 + scale_mlp.unsqueeze(1)) + shift_mlp.unsqueeze(1)
71
+ h = self.mlp(h)
72
+ h = h * gate_mlp.unsqueeze(1)
73
+ x = x + h
74
+ return x
75
+
76
+ def forward(self, x: torch.Tensor, mod: torch.Tensor) -> torch.Tensor:
77
+ if self.use_checkpoint:
78
+ return torch.utils.checkpoint.checkpoint(
79
+ self._forward, x, mod, use_reentrant=False
80
+ )
81
+ else:
82
+ return self._forward(x, mod)
83
+
84
+
85
+ class ModulatedTransformerCrossBlock(nn.Module):
86
+ """
87
+ Transformer cross-attention block (MSA + MCA + FFN) with adaptive layer norm conditioning.
88
+ """
89
+
90
+ def __init__(
91
+ self,
92
+ channels: int,
93
+ ctx_channels: int,
94
+ num_heads: int,
95
+ mlp_ratio: float = 4.0,
96
+ attn_mode: Literal["full", "windowed"] = "full",
97
+ window_size: Optional[int] = None,
98
+ shift_window: Optional[Tuple[int, int, int]] = None,
99
+ use_checkpoint: bool = False,
100
+ use_rope: bool = False,
101
+ qk_rms_norm: bool = False,
102
+ qk_rms_norm_cross: bool = False,
103
+ qkv_bias: bool = True,
104
+ share_mod: bool = False,
105
+ ):
106
+ super().__init__()
107
+ self.use_checkpoint = use_checkpoint
108
+ self.share_mod = share_mod
109
+ self.norm1 = LayerNorm32(channels, elementwise_affine=False, eps=1e-6)
110
+ self.norm2 = LayerNorm32(channels, elementwise_affine=True, eps=1e-6)
111
+ self.norm3 = LayerNorm32(channels, elementwise_affine=False, eps=1e-6)
112
+ self.self_attn = MultiHeadAttention(
113
+ channels,
114
+ num_heads=num_heads,
115
+ type="self",
116
+ attn_mode=attn_mode,
117
+ window_size=window_size,
118
+ shift_window=shift_window,
119
+ qkv_bias=qkv_bias,
120
+ use_rope=use_rope,
121
+ qk_rms_norm=qk_rms_norm,
122
+ )
123
+ self.cross_attn = MultiHeadAttention(
124
+ channels,
125
+ ctx_channels=ctx_channels,
126
+ num_heads=num_heads,
127
+ type="cross",
128
+ attn_mode="full",
129
+ qkv_bias=qkv_bias,
130
+ qk_rms_norm=qk_rms_norm_cross,
131
+ )
132
+ self.mlp = FeedForwardNet(
133
+ channels,
134
+ mlp_ratio=mlp_ratio,
135
+ )
136
+ if not share_mod:
137
+ self.adaLN_modulation = nn.Sequential(
138
+ nn.SiLU(), nn.Linear(channels, 6 * channels, bias=True)
139
+ )
140
+
141
+ def _forward(self, x: torch.Tensor, mod: torch.Tensor, context: torch.Tensor):
142
+ if self.share_mod:
143
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mod.chunk(
144
+ 6, dim=1
145
+ )
146
+ else:
147
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
148
+ self.adaLN_modulation(mod).chunk(6, dim=1)
149
+ )
150
+ h = self.norm1(x)
151
+ h = h * (1 + scale_msa.unsqueeze(1)) + shift_msa.unsqueeze(1)
152
+ h = self.self_attn(h)
153
+ h = h * gate_msa.unsqueeze(1)
154
+ x = x + h
155
+ h = self.norm2(x)
156
+ h = self.cross_attn(h, context)
157
+ x = x + h
158
+ h = self.norm3(x)
159
+ h = h * (1 + scale_mlp.unsqueeze(1)) + shift_mlp.unsqueeze(1)
160
+ h = self.mlp(h)
161
+ h = h * gate_mlp.unsqueeze(1)
162
+ x = x + h
163
+ return x
164
+
165
+ def forward(self, x: torch.Tensor, mod: torch.Tensor, context: torch.Tensor):
166
+ if self.use_checkpoint:
167
+ return torch.utils.checkpoint.checkpoint(
168
+ self._forward, x, mod, context, use_reentrant=False
169
+ )
170
+ else:
171
+ return self._forward(x, mod, context)
172
+
173
+
174
+ class MOTModulatedTransformerCrossBlock(nn.Module):
175
+ """
176
+ Transformer cross-attention block (MSA + MCA + FFN) with adaptive layer norm conditioning.
177
+ """
178
+
179
+ def __init__(
180
+ self,
181
+ channels: int,
182
+ ctx_channels: int,
183
+ num_heads: int,
184
+ mlp_ratio: float = 4.0,
185
+ attn_mode: Literal["full", "windowed"] = "full",
186
+ window_size: Optional[int] = None,
187
+ shift_window: Optional[Tuple[int, int, int]] = None,
188
+ use_checkpoint: bool = False,
189
+ use_rope: bool = False,
190
+ qk_rms_norm: bool = False,
191
+ qk_rms_norm_cross: bool = False,
192
+ qkv_bias: bool = True,
193
+ share_mod: bool = False,
194
+ latent_names: List = None,
195
+ freeze_shared_parameters: bool = False,
196
+ ):
197
+ super().__init__()
198
+ self.use_checkpoint = use_checkpoint
199
+ self.share_mod = share_mod
200
+ self.norm1 = torch.nn.ModuleDict(
201
+ {
202
+ latent_name: LayerNorm32(channels, elementwise_affine=False, eps=1e-6)
203
+ for latent_name in latent_names
204
+ }
205
+ )
206
+ self.norm2 = torch.nn.ModuleDict(
207
+ {
208
+ latent_name: LayerNorm32(channels, elementwise_affine=True, eps=1e-6)
209
+ for latent_name in latent_names
210
+ }
211
+ )
212
+ self.norm3 = torch.nn.ModuleDict(
213
+ {
214
+ latent_name: LayerNorm32(channels, elementwise_affine=False, eps=1e-6)
215
+ for latent_name in latent_names
216
+ }
217
+ )
218
+ self.self_attn = MOTMultiHeadSelfAttention(
219
+ channels,
220
+ num_heads=num_heads,
221
+ type="self",
222
+ attn_mode=attn_mode,
223
+ window_size=window_size,
224
+ shift_window=shift_window,
225
+ qkv_bias=qkv_bias,
226
+ use_rope=use_rope,
227
+ qk_rms_norm=qk_rms_norm,
228
+ latent_names=latent_names,
229
+ )
230
+ self.cross_attn = torch.nn.ModuleDict(
231
+ {
232
+ latent_name: MultiHeadAttention(
233
+ channels,
234
+ ctx_channels=ctx_channels,
235
+ num_heads=num_heads,
236
+ type="cross",
237
+ attn_mode="full",
238
+ qkv_bias=qkv_bias,
239
+ qk_rms_norm=qk_rms_norm_cross,
240
+ )
241
+ for latent_name in latent_names
242
+ }
243
+ )
244
+ self.mlp = torch.nn.ModuleDict(
245
+ {
246
+ latent_name: FeedForwardNet(
247
+ channels,
248
+ mlp_ratio=mlp_ratio,
249
+ )
250
+ for latent_name in latent_names
251
+ }
252
+ )
253
+ if not share_mod:
254
+ self.adaLN_modulation = nn.Sequential(
255
+ nn.SiLU(), nn.Linear(channels, 6 * channels, bias=True)
256
+ )
257
+ if freeze_shared_parameters:
258
+ self.adaLN_modulation.eval()
259
+ self.adaLN_modulation.requires_grad_(False)
260
+
261
+ def _apply_module(self, h, module):
262
+ return module(h)
263
+
264
+ def _apply_cross_attn(self, h, cross_attn, context):
265
+ return cross_attn(h, context)
266
+
267
+ def _apply_msa(self, h, scale_msa, shift_msa):
268
+ return h * (1 + scale_msa.unsqueeze(1)) + shift_msa.unsqueeze(1)
269
+
270
+ def _apply_mlp(self, h, scale_mlp, shift_mlp):
271
+ return h * (1 + scale_mlp.unsqueeze(1)) + shift_mlp.unsqueeze(1)
272
+
273
+ def _apply_add(self, x, h):
274
+ return x + h
275
+
276
+ def _apply_multiplication(self, h, multiplier):
277
+ return h * multiplier.unsqueeze(1)
278
+
279
+ # This is stupid, _pytree does not support ModuleDict
280
+ def _moduledict_to_dict(self, module):
281
+ return {key: module for key, module in module.items()}
282
+
283
+ def _forward(self, x: torch.Tensor, mod: torch.Tensor, context: torch.Tensor, mode="default"):
284
+ if self.share_mod:
285
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mod.chunk(
286
+ 6, dim=1
287
+ )
288
+ else:
289
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
290
+ self.adaLN_modulation(mod).chunk(6, dim=1)
291
+ )
292
+ h = _pytree.tree_map(self._apply_module, x, self._moduledict_to_dict(self.norm1))
293
+ h = _pytree.tree_map(
294
+ partial(self._apply_msa, scale_msa=scale_msa, shift_msa=shift_msa),
295
+ h
296
+ )
297
+ h = self.self_attn(h, mode=mode)
298
+ h = _pytree.tree_map(
299
+ partial(self._apply_multiplication, multiplier=gate_msa),
300
+ h
301
+ )
302
+ x = _pytree.tree_map(
303
+ self._apply_add,
304
+ x,
305
+ h
306
+ )
307
+ h = _pytree.tree_map(self._apply_module, x, self._moduledict_to_dict(self.norm2))
308
+ h = _pytree.tree_map(
309
+ partial(self._apply_cross_attn, context=context),
310
+ h,
311
+ self._moduledict_to_dict(self.cross_attn),
312
+ )
313
+ x = _pytree.tree_map(
314
+ self._apply_add,
315
+ x,
316
+ h
317
+ )
318
+ h = _pytree.tree_map(self._apply_module, x, self._moduledict_to_dict(self.norm3))
319
+ h = _pytree.tree_map(
320
+ partial(self._apply_mlp, scale_mlp=scale_mlp, shift_mlp=shift_mlp),
321
+ h
322
+ )
323
+ h = _pytree.tree_map(self._apply_module, h, self._moduledict_to_dict(self.mlp))
324
+ h = _pytree.tree_map(
325
+ partial(self._apply_multiplication, multiplier=gate_mlp),
326
+ h
327
+ )
328
+ x = _pytree.tree_map(
329
+ self._apply_add,
330
+ x,
331
+ h
332
+ )
333
+ return x
334
+
335
+ def forward(self, x: Dict, mod: torch.Tensor, context: torch.Tensor, mode="default"):
336
+ if self.use_checkpoint:
337
+ return torch.utils.checkpoint.checkpoint(
338
+ self._forward, x, mod, context, mode, use_reentrant=False
339
+ )
340
+ else:
341
+ return self._forward(x, mod, context, mode=mode)
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/utils.py ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ import torch.nn as nn
3
+ import torch
4
+
5
+ FP16_TYPE = torch.float16
6
+
7
+ FP16_MODULES = [
8
+ nn.Conv1d,
9
+ nn.Conv2d,
10
+ nn.Conv3d,
11
+ nn.ConvTranspose1d,
12
+ nn.ConvTranspose2d,
13
+ nn.ConvTranspose3d,
14
+ nn.Linear,
15
+ ]
16
+
17
+ # If we add sparse modules back in, they are compatible with FP16.
18
+ # But for now we don't have them and avoid the dependency on FlashAttention
19
+ # Instead using the torch implementation of FlashAttention in SDPA.
20
+ try:
21
+ from ..modules import sparse as sp
22
+
23
+ FP16_MODULES += [
24
+ sp.SparseConv3d,
25
+ sp.SparseInverseConv3d,
26
+ sp.SparseLinear,
27
+ ]
28
+ except ImportError:
29
+ pass
30
+
31
+ FP16_MODULES = tuple(FP16_MODULES)
32
+
33
+
34
+ def convert_module_to_f16(l):
35
+ """
36
+ Convert primitive modules to float16.
37
+ """
38
+ if isinstance(l, FP16_MODULES):
39
+ for p in l.parameters():
40
+ # p.data = p.data.half()
41
+ p.data = p.data.to(FP16_TYPE)
42
+
43
+
44
+ def convert_module_to_f32(l):
45
+ """
46
+ Convert primitive modules to float32, undoing convert_module_to_f16().
47
+ """
48
+ if isinstance(l, FP16_MODULES):
49
+ for p in l.parameters():
50
+ p.data = p.data.float()
51
+
52
+
53
+ def zero_module(module):
54
+ """
55
+ Zero out the parameters of a module and return it.
56
+ """
57
+ for p in module.parameters():
58
+ p.detach().zero_()
59
+ return module
60
+
61
+
62
+ def scale_module(module, scale):
63
+ """
64
+ Scale the parameters of a module and return it.
65
+ """
66
+ for p in module.parameters():
67
+ p.detach().mul_(scale)
68
+ return module
69
+
70
+
71
+ def modulate(x, shift, scale):
72
+ return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1)
migrator/code/slat_dev_20261111/sam3d_objects/model/io.py ADDED
@@ -0,0 +1,210 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ from typing import Any, Callable, Dict, List, Optional, Union, Iterable
3
+ import lightning.pytorch as pl
4
+ import torch
5
+ from pathlib import Path
6
+ import os
7
+ import re
8
+ from loguru import logger
9
+ from lightning.pytorch.utilities.consolidate_checkpoint import (
10
+ _format_checkpoint,
11
+ _load_distributed_checkpoint,
12
+ )
13
+ from glob import glob
14
+
15
+ from sam3d_objects.data.utils import get_child, set_child
16
+
17
+
18
+ def rename_checkpoint_weights_using_suffix_matching(
19
+ checkpoint_path_in,
20
+ checkpoint_path_out,
21
+ model: torch.nn.Module,
22
+ strict: bool = True,
23
+ keys: Optional[List[Any]] = (),
24
+ ):
25
+ # extract model names
26
+ param_names = [n for n, _ in model.named_parameters()]
27
+ buffer_names = [n for n, _ in model.named_buffers()]
28
+ model_names = param_names + buffer_names
29
+
30
+ # load stored weights
31
+ state = torch.load(checkpoint_path_in, weights_only=False)
32
+
33
+ model_state = get_child(state, *keys)
34
+ model_state_names = list(model_state.keys())
35
+
36
+ # sort reversed names (sort by suffix)
37
+ model_names_rev = sorted([n[::-1] for n in model_names])
38
+ model_state_names_rev = sorted([n[::-1] for n in model_state_names])
39
+
40
+ if strict and len(model_names) != len(model_state_names):
41
+ raise RuntimeError(
42
+ f"model and state don't have the same number of parameters ({len(model_names)} != {len(model_state_names)}), cannot match them (set strict = False to relax constraint)"
43
+ )
44
+
45
+ def common_prefix_length(str_0: str, str_1: str):
46
+ for count in range(min(len(str_0), len(str_1))):
47
+ if str_0[count] != str_1[count]:
48
+ break
49
+ return count
50
+
51
+ # attempt to match every model names to largest suffic matched weight
52
+ name_mapping = {}
53
+ i, j = 0, 0
54
+ last_n = 0
55
+ while i < len(model_names_rev):
56
+ if j < len(model_state_names_rev):
57
+ n = common_prefix_length(model_names_rev[i], model_state_names_rev[j])
58
+ else:
59
+ n = 0
60
+
61
+ if n >= last_n:
62
+ last_n = n
63
+ j += 1
64
+ else:
65
+ last_n = 0
66
+ name_mapping[model_names_rev[i][::-1]] = model_state_names_rev[j - 1][::-1]
67
+ i += 1
68
+
69
+ if not j < len(model_state_names_rev) + 1:
70
+ break
71
+
72
+ # not all names might have been matched
73
+ if i < len(model_names):
74
+ raise RuntimeError("could not suffix match parameter names")
75
+
76
+ for k, v in name_mapping.items():
77
+ logger.debug(f"{k} <- {v}")
78
+
79
+ # rename weights according to matches and save to disk
80
+ model_state_out = {k: model_state[v] for k, v in name_mapping.items()}
81
+ set_child(state, model_state_out, *keys)
82
+ torch.save(state, checkpoint_path_out)
83
+
84
+
85
+ def remove_prefix_state_dict_fn(prefix: str):
86
+ n = len(prefix)
87
+
88
+ def state_dict_fn(state_dict):
89
+ return {
90
+ (key[n:] if key.startswith(prefix) else key): value
91
+ for key, value in state_dict.items()
92
+ }
93
+
94
+ return state_dict_fn
95
+
96
+
97
+ def add_prefix_state_dict_fn(prefix: str):
98
+ def state_dict_fn(state_dict):
99
+ return {prefix + key: value for key, value in state_dict.items()}
100
+
101
+ return state_dict_fn
102
+
103
+
104
+ def filter_and_remove_prefix_state_dict_fn(prefix: str):
105
+ n = len(prefix)
106
+
107
+ def state_dict_fn(state_dict):
108
+ return {
109
+ key[n:]: value
110
+ for key, value in state_dict.items()
111
+ if key.startswith(prefix)
112
+ }
113
+
114
+ return state_dict_fn
115
+
116
+
117
+ def get_last_checkpoint(path: str):
118
+ checkpoints = glob(os.path.join(path, "epoch=*-step=*.ckpt"))
119
+ prog = re.compile(r"epoch=(\d+)-step=(\d+).ckpt")
120
+
121
+ checkpoints_to_sort = []
122
+ for checkpoint in checkpoints:
123
+ checkpoint_name = os.path.basename(checkpoint)
124
+ match = prog.match(checkpoint_name)
125
+ if match is not None:
126
+ n_epoch, n_step = prog.match(checkpoint_name).groups()
127
+ n_epoch, n_step = int(n_epoch), int(n_step)
128
+ checkpoints_to_sort.append((n_epoch, n_step, checkpoint))
129
+
130
+ sorted_checkpoints = sorted(checkpoints_to_sort)
131
+ if not len(sorted_checkpoints) > 0:
132
+ raise RuntimeError(f"no checkpoint has been found at path : {path}")
133
+ return sorted_checkpoints[-1][2]
134
+
135
+
136
+ def load_sharded_checkpoint(path: str, device: Optional[str]):
137
+ if device != "cpu":
138
+ raise RuntimeError(
139
+ f'loading sharded weights on device "{device}" is not available, please use the "cpu" device instead'
140
+ )
141
+ checkpoint = _load_distributed_checkpoint(Path(path))
142
+ checkpoint = _format_checkpoint(checkpoint)
143
+ return checkpoint
144
+
145
+
146
+ def load_model_from_checkpoint(
147
+ model: Union[pl.LightningModule, torch.nn.Module],
148
+ checkpoint_path: str,
149
+ strict: bool = True,
150
+ device: Optional[str] = None,
151
+ freeze: bool = False,
152
+ eval: bool = False,
153
+ map_name: Union[Dict[str, str], None] = None,
154
+ remove_name: Union[List[str], None] = None,
155
+ state_dict_key: Union[None, str, Iterable[str]] = "state_dict",
156
+ state_dict_fn: Optional[Callable[[Any], Any]] = None,
157
+ ):
158
+ logger.info(f"Loading checkpoint from {checkpoint_path}")
159
+ if os.path.isfile(checkpoint_path):
160
+ checkpoint = torch.load(
161
+ checkpoint_path,
162
+ map_location=device,
163
+ weights_only=False,
164
+ )
165
+ elif os.path.isdir(checkpoint_path): # sharded
166
+ checkpoint = load_sharded_checkpoint(checkpoint_path, device=device)
167
+ else: # if neither a file nor a directory, path does not exist
168
+ raise FileNotFoundError(checkpoint_path)
169
+
170
+ if isinstance(model, pl.LightningModule):
171
+ model.on_load_checkpoint(checkpoint)
172
+
173
+ # get state dictionary
174
+ state_dict = checkpoint
175
+ if state_dict_key is not None:
176
+ if isinstance(state_dict_key, str):
177
+ state_dict_key = (state_dict_key,)
178
+ state_dict = get_child(state_dict, *state_dict_key)
179
+
180
+ # remove names
181
+ if remove_name is not None:
182
+ for name in remove_name:
183
+ del state_dict[name]
184
+
185
+ # remap names
186
+ if map_name is not None:
187
+ for src, dst in map_name.items():
188
+ if src not in state_dict:
189
+ continue
190
+ state_dict[dst] = state_dict[src]
191
+ del state_dict[src]
192
+
193
+ # apply custom changes to dict
194
+ if state_dict_fn is not None:
195
+ state_dict = state_dict_fn(state_dict)
196
+
197
+ model.load_state_dict(state_dict, strict=strict)
198
+
199
+ if device is not None:
200
+ model = model.to(device)
201
+
202
+ if freeze:
203
+ for param in model.parameters():
204
+ param.requires_grad = False
205
+ eval = True
206
+
207
+ if eval:
208
+ model.eval()
209
+
210
+ return model
migrator/code/slat_dev_20261111/sam3d_objects/model/layers/__init__.py ADDED
File without changes
migrator/code/slat_dev_20261111/sam3d_objects/model/layers/llama3/__init__.py ADDED
File without changes
migrator/code/slat_dev_20261111/sam3d_objects/model/layers/llama3/ff.py ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ from torch import nn
3
+ import torch.nn.functional as F
4
+ from typing import Optional
5
+
6
+
7
+ class FeedForward(nn.Module):
8
+ def __init__(
9
+ self,
10
+ dim: int,
11
+ hidden_dim: int,
12
+ multiple_of: int = 256,
13
+ ffn_dim_multiplier: Optional[float] = None,
14
+ output_dim: Optional[int] = None,
15
+ skip_w2: bool = False,
16
+ ):
17
+ """
18
+ Llama3 FeedForward layer
19
+ https://github.com/meta-llama/llama3/blob/a0940f9cf7065d45bb6675660f80d305c041a754/llama/model.py#L193
20
+ """
21
+ super().__init__()
22
+ hidden_dim = int(2 * hidden_dim / 3)
23
+ # custom dim factor multiplier
24
+ if ffn_dim_multiplier is not None:
25
+ hidden_dim = int(ffn_dim_multiplier * hidden_dim)
26
+ hidden_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of)
27
+
28
+ if output_dim is None:
29
+ output_dim = dim
30
+
31
+ self.skip_w2 = skip_w2
32
+ self.w1 = nn.Linear(dim, hidden_dim, bias=False)
33
+ if not self.skip_w2:
34
+ self.w2 = nn.Linear(hidden_dim, output_dim, bias=False)
35
+ self.w3 = nn.Linear(dim, hidden_dim, bias=False)
36
+
37
+ def forward(self, x):
38
+ x = F.silu(self.w1(x)) * self.w3(x)
39
+ if self.skip_w2:
40
+ return x
41
+ return self.w2(x)