Add slat_dev snapshot 2026-11-11: stage-2 code with test-time soft-mask bias on/off (BIAS_INFERENCE.md) (part 2)
Browse files- migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/attention/windowed_attn.py +192 -0
- migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/basic.py +569 -0
- migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/conv/__init__.py +27 -0
- migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/conv/conv_spconv.py +139 -0
- migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/linear.py +14 -0
- migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/nonlinearity.py +30 -0
- migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/norm.py +64 -0
- migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/spatial.py +131 -0
- migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/transformer/__init__.py +3 -0
- migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/transformer/blocks.py +162 -0
- migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/transformer/modulated.py +186 -0
- migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/spatial.py +65 -0
- migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/transformer/__init__.py +3 -0
- migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/transformer/blocks.py +197 -0
- migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/transformer/modulated.py +341 -0
- migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/utils.py +72 -0
- migrator/code/slat_dev_20261111/sam3d_objects/model/io.py +210 -0
- migrator/code/slat_dev_20261111/sam3d_objects/model/layers/__init__.py +0 -0
- migrator/code/slat_dev_20261111/sam3d_objects/model/layers/llama3/__init__.py +0 -0
- migrator/code/slat_dev_20261111/sam3d_objects/model/layers/llama3/ff.py +41 -0
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/attention/windowed_attn.py
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
from typing import *
|
| 3 |
+
import torch
|
| 4 |
+
import math
|
| 5 |
+
from .. import SparseTensor
|
| 6 |
+
from .. import DEBUG, ATTN
|
| 7 |
+
|
| 8 |
+
if ATTN == "xformers":
|
| 9 |
+
import xformers.ops as xops
|
| 10 |
+
elif ATTN == "flash_attn":
|
| 11 |
+
import flash_attn
|
| 12 |
+
elif ATTN == "sdpa":
|
| 13 |
+
from torch.nn.functional import scaled_dot_product_attention as sdpa
|
| 14 |
+
from .masked_sdpa import masked_sdpa
|
| 15 |
+
else:
|
| 16 |
+
raise ValueError(f"Unknown attention module: {ATTN}")
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
__all__ = [
|
| 20 |
+
"sparse_windowed_scaled_dot_product_self_attention",
|
| 21 |
+
]
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def calc_window_partition(
|
| 25 |
+
tensor: SparseTensor,
|
| 26 |
+
window_size: Union[int, Tuple[int, ...]],
|
| 27 |
+
shift_window: Union[int, Tuple[int, ...]] = 0,
|
| 28 |
+
) -> Tuple[torch.Tensor, torch.Tensor, List[int], List[int]]:
|
| 29 |
+
"""
|
| 30 |
+
Calculate serialization and partitioning for a set of coordinates.
|
| 31 |
+
|
| 32 |
+
Args:
|
| 33 |
+
tensor (SparseTensor): The input tensor.
|
| 34 |
+
window_size (int): The window size to use.
|
| 35 |
+
shift_window (Tuple[int, ...]): The shift of serialized coordinates.
|
| 36 |
+
|
| 37 |
+
Returns:
|
| 38 |
+
(torch.Tensor): Forwards indices.
|
| 39 |
+
(torch.Tensor): Backwards indices.
|
| 40 |
+
(List[int]): Sequence lengths.
|
| 41 |
+
(List[int]): Sequence batch indices.
|
| 42 |
+
"""
|
| 43 |
+
DIM = tensor.coords.shape[1] - 1
|
| 44 |
+
shift_window = (
|
| 45 |
+
(shift_window,) * DIM if isinstance(shift_window, int) else shift_window
|
| 46 |
+
)
|
| 47 |
+
window_size = (window_size,) * DIM if isinstance(window_size, int) else window_size
|
| 48 |
+
shifted_coords = tensor.coords.clone().detach()
|
| 49 |
+
shifted_coords[:, 1:] += torch.tensor(
|
| 50 |
+
shift_window, device=tensor.device, dtype=torch.int32
|
| 51 |
+
).unsqueeze(0)
|
| 52 |
+
|
| 53 |
+
MAX_COORDS = shifted_coords[:, 1:].max(dim=0).values.tolist()
|
| 54 |
+
NUM_WINDOWS = [math.ceil((mc + 1) / ws) for mc, ws in zip(MAX_COORDS, window_size)]
|
| 55 |
+
OFFSET = torch.cumprod(torch.tensor([1] + NUM_WINDOWS[::-1]), dim=0).tolist()[::-1]
|
| 56 |
+
|
| 57 |
+
shifted_coords[:, 1:] //= torch.tensor(
|
| 58 |
+
window_size, device=tensor.device, dtype=torch.int32
|
| 59 |
+
).unsqueeze(0)
|
| 60 |
+
shifted_indices = (
|
| 61 |
+
shifted_coords
|
| 62 |
+
* torch.tensor(OFFSET, device=tensor.device, dtype=torch.int32).unsqueeze(0)
|
| 63 |
+
).sum(dim=1)
|
| 64 |
+
fwd_indices = torch.argsort(shifted_indices)
|
| 65 |
+
bwd_indices = torch.empty_like(fwd_indices)
|
| 66 |
+
bwd_indices[fwd_indices] = torch.arange(fwd_indices.shape[0], device=tensor.device)
|
| 67 |
+
seq_lens = torch.bincount(shifted_indices)
|
| 68 |
+
seq_batch_indices = (
|
| 69 |
+
torch.arange(seq_lens.shape[0], device=tensor.device, dtype=torch.int32)
|
| 70 |
+
// OFFSET[0]
|
| 71 |
+
)
|
| 72 |
+
mask = seq_lens != 0
|
| 73 |
+
seq_lens = seq_lens[mask].tolist()
|
| 74 |
+
seq_batch_indices = seq_batch_indices[mask].tolist()
|
| 75 |
+
|
| 76 |
+
return fwd_indices, bwd_indices, seq_lens, seq_batch_indices
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def sparse_windowed_scaled_dot_product_self_attention(
|
| 80 |
+
qkv: SparseTensor, window_size: int, shift_window: Tuple[int, int, int] = (0, 0, 0)
|
| 81 |
+
) -> SparseTensor:
|
| 82 |
+
"""
|
| 83 |
+
Apply windowed scaled dot product self attention to a sparse tensor.
|
| 84 |
+
|
| 85 |
+
Args:
|
| 86 |
+
qkv (SparseTensor): [N, *, 3, H, C] sparse tensor containing Qs, Ks, and Vs.
|
| 87 |
+
window_size (int): The window size to use.
|
| 88 |
+
shift_window (Tuple[int, int, int]): The shift of serialized coordinates.
|
| 89 |
+
shift (int): The shift to use.
|
| 90 |
+
"""
|
| 91 |
+
assert (
|
| 92 |
+
len(qkv.shape) == 4 and qkv.shape[1] == 3
|
| 93 |
+
), f"Invalid shape for qkv, got {qkv.shape}, expected [N, *, 3, H, C]"
|
| 94 |
+
|
| 95 |
+
serialization_spatial_cache_name = f"window_partition_{window_size}_{shift_window}"
|
| 96 |
+
serialization_spatial_cache = qkv.get_spatial_cache(
|
| 97 |
+
serialization_spatial_cache_name
|
| 98 |
+
)
|
| 99 |
+
if serialization_spatial_cache is None:
|
| 100 |
+
fwd_indices, bwd_indices, seq_lens, seq_batch_indices = calc_window_partition(
|
| 101 |
+
qkv, window_size, shift_window
|
| 102 |
+
)
|
| 103 |
+
qkv.register_spatial_cache(
|
| 104 |
+
serialization_spatial_cache_name,
|
| 105 |
+
(fwd_indices, bwd_indices, seq_lens, seq_batch_indices),
|
| 106 |
+
)
|
| 107 |
+
else:
|
| 108 |
+
fwd_indices, bwd_indices, seq_lens, seq_batch_indices = (
|
| 109 |
+
serialization_spatial_cache
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
M = fwd_indices.shape[0]
|
| 113 |
+
T = qkv.feats.shape[0]
|
| 114 |
+
H = qkv.feats.shape[2]
|
| 115 |
+
C = qkv.feats.shape[3]
|
| 116 |
+
|
| 117 |
+
qkv_feats = qkv.feats[fwd_indices] # [M, 3, H, C]
|
| 118 |
+
|
| 119 |
+
if DEBUG:
|
| 120 |
+
start = 0
|
| 121 |
+
qkv_coords = qkv.coords[fwd_indices]
|
| 122 |
+
for i in range(len(seq_lens)):
|
| 123 |
+
seq_coords = qkv_coords[start : start + seq_lens[i]]
|
| 124 |
+
assert (
|
| 125 |
+
seq_coords[:, 0] == seq_batch_indices[i]
|
| 126 |
+
).all(), (
|
| 127 |
+
f"SparseWindowedScaledDotProductSelfAttention: batch index mismatch"
|
| 128 |
+
)
|
| 129 |
+
assert (
|
| 130 |
+
seq_coords[:, 1:].max(dim=0).values
|
| 131 |
+
- seq_coords[:, 1:].min(dim=0).values
|
| 132 |
+
< window_size
|
| 133 |
+
).all(), (
|
| 134 |
+
f"SparseWindowedScaledDotProductSelfAttention: window size exceeded"
|
| 135 |
+
)
|
| 136 |
+
start += seq_lens[i]
|
| 137 |
+
|
| 138 |
+
if all([seq_len == window_size for seq_len in seq_lens]):
|
| 139 |
+
B = len(seq_lens)
|
| 140 |
+
N = window_size
|
| 141 |
+
qkv_feats = qkv_feats.reshape(B, N, 3, H, C)
|
| 142 |
+
if ATTN == "xformers":
|
| 143 |
+
q, k, v = qkv_feats.unbind(dim=2) # [B, N, H, C]
|
| 144 |
+
out = xops.memory_efficient_attention(q, k, v) # [B, N, H, C]
|
| 145 |
+
elif ATTN == "flash_attn":
|
| 146 |
+
out = flash_attn.flash_attn_qkvpacked_func(qkv_feats) # [B, N, H, C]
|
| 147 |
+
elif ATTN == "sdpa":
|
| 148 |
+
q, k, v = qkv_feats.unbind(dim=2)
|
| 149 |
+
q = q.permute(0, 2, 1, 3) # [N, H, L, C]
|
| 150 |
+
k = k.permute(0, 2, 1, 3) # [N, H, L, C]
|
| 151 |
+
v = v.permute(0, 2, 1, 3) # [N, H, L, C]
|
| 152 |
+
out = sdpa(q, k, v) # [N, H, L, C]
|
| 153 |
+
out = out.permute(0, 2, 1, 3) # [N, L, H, C]
|
| 154 |
+
else:
|
| 155 |
+
raise ValueError(f"Unknown attention module: {ATTN}")
|
| 156 |
+
out = out.reshape(B * N, H, C) # [M, H, C]
|
| 157 |
+
else:
|
| 158 |
+
if ATTN == "xformers":
|
| 159 |
+
q, k, v = qkv_feats.unbind(dim=1) # [M, H, C]
|
| 160 |
+
q = q.unsqueeze(0) # [1, M, H, C]
|
| 161 |
+
k = k.unsqueeze(0) # [1, M, H, C]
|
| 162 |
+
v = v.unsqueeze(0) # [1, M, H, C]
|
| 163 |
+
mask = xops.fmha.BlockDiagonalMask.from_seqlens(seq_lens)
|
| 164 |
+
out = xops.memory_efficient_attention(q, k, v, mask)[0] # [M, H, C]
|
| 165 |
+
elif ATTN == "flash_attn":
|
| 166 |
+
cu_seqlens = (
|
| 167 |
+
torch.cat(
|
| 168 |
+
[torch.tensor([0]), torch.cumsum(torch.tensor(seq_lens), dim=0)],
|
| 169 |
+
dim=0,
|
| 170 |
+
)
|
| 171 |
+
.to(qkv.device)
|
| 172 |
+
.int()
|
| 173 |
+
)
|
| 174 |
+
out = flash_attn.flash_attn_varlen_qkvpacked_func(
|
| 175 |
+
qkv_feats, cu_seqlens, max(seq_lens)
|
| 176 |
+
) # [M, H, C]
|
| 177 |
+
elif ATTN == "sdpa":
|
| 178 |
+
q, k, v = qkv_feats.unbind(dim=1)
|
| 179 |
+
q = q.unsqueeze(0)
|
| 180 |
+
k = k.unsqueeze(0)
|
| 181 |
+
v = v.unsqueeze(0)
|
| 182 |
+
out = masked_sdpa(q, k, v, seq_lens, seq_lens)
|
| 183 |
+
|
| 184 |
+
out = out[bwd_indices] # [T, H, C]
|
| 185 |
+
|
| 186 |
+
if DEBUG:
|
| 187 |
+
qkv_coords = qkv_coords[bwd_indices]
|
| 188 |
+
assert torch.equal(
|
| 189 |
+
qkv_coords, qkv.coords
|
| 190 |
+
), "SparseWindowedScaledDotProductSelfAttention: coordinate mismatch"
|
| 191 |
+
|
| 192 |
+
return qkv.replace(out)
|
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/basic.py
ADDED
|
@@ -0,0 +1,569 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
from typing import *
|
| 3 |
+
import torch
|
| 4 |
+
from . import BACKEND, DEBUG
|
| 5 |
+
|
| 6 |
+
if BACKEND == "torchsparse":
|
| 7 |
+
from torchsparse import SparseTensor as SparseTensorData
|
| 8 |
+
elif BACKEND == "spconv":
|
| 9 |
+
from spconv.pytorch import SparseConvTensor as SparseTensorData
|
| 10 |
+
|
| 11 |
+
__all__ = [
|
| 12 |
+
"SparseTensor",
|
| 13 |
+
"sparse_batch_broadcast",
|
| 14 |
+
"sparse_batch_op",
|
| 15 |
+
"sparse_cat",
|
| 16 |
+
"sparse_unbind",
|
| 17 |
+
]
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class SparseTensor:
|
| 21 |
+
"""
|
| 22 |
+
Sparse tensor with support for both torchsparse and spconv backends.
|
| 23 |
+
|
| 24 |
+
Parameters:
|
| 25 |
+
- feats (torch.Tensor): Features of the sparse tensor.
|
| 26 |
+
- coords (torch.Tensor): Coordinates of the sparse tensor.
|
| 27 |
+
- shape (torch.Size): Shape of the sparse tensor.
|
| 28 |
+
- layout (List[slice]): Layout of the sparse tensor for each batch
|
| 29 |
+
- data (SparseTensorData): Sparse tensor data used for convolusion
|
| 30 |
+
|
| 31 |
+
NOTE:
|
| 32 |
+
- Data corresponding to a same batch should be contiguous.
|
| 33 |
+
- Coords should be in [0, 1023]
|
| 34 |
+
"""
|
| 35 |
+
|
| 36 |
+
@overload
|
| 37 |
+
def __init__(
|
| 38 |
+
self,
|
| 39 |
+
feats: torch.Tensor,
|
| 40 |
+
coords: torch.Tensor,
|
| 41 |
+
shape: Optional[torch.Size] = None,
|
| 42 |
+
layout: Optional[List[slice]] = None,
|
| 43 |
+
**kwargs,
|
| 44 |
+
): ...
|
| 45 |
+
|
| 46 |
+
@overload
|
| 47 |
+
def __init__(
|
| 48 |
+
self,
|
| 49 |
+
data,
|
| 50 |
+
shape: Optional[torch.Size] = None,
|
| 51 |
+
layout: Optional[List[slice]] = None,
|
| 52 |
+
**kwargs,
|
| 53 |
+
): ...
|
| 54 |
+
|
| 55 |
+
def __init__(self, *args, **kwargs):
|
| 56 |
+
method_id = 0
|
| 57 |
+
if len(args) != 0:
|
| 58 |
+
method_id = 0 if isinstance(args[0], torch.Tensor) else 1
|
| 59 |
+
else:
|
| 60 |
+
method_id = 1 if "data" in kwargs else 0
|
| 61 |
+
|
| 62 |
+
if method_id == 0:
|
| 63 |
+
feats, coords, shape, layout = args + (None,) * (4 - len(args))
|
| 64 |
+
if "feats" in kwargs:
|
| 65 |
+
feats = kwargs["feats"]
|
| 66 |
+
del kwargs["feats"]
|
| 67 |
+
if "coords" in kwargs:
|
| 68 |
+
coords = kwargs["coords"]
|
| 69 |
+
del kwargs["coords"]
|
| 70 |
+
if "shape" in kwargs:
|
| 71 |
+
shape = kwargs["shape"]
|
| 72 |
+
del kwargs["shape"]
|
| 73 |
+
if "layout" in kwargs:
|
| 74 |
+
layout = kwargs["layout"]
|
| 75 |
+
del kwargs["layout"]
|
| 76 |
+
|
| 77 |
+
if shape is None:
|
| 78 |
+
shape = self.__cal_shape(feats, coords)
|
| 79 |
+
if layout is None:
|
| 80 |
+
layout = self.__cal_layout(coords, shape[0])
|
| 81 |
+
if BACKEND == "torchsparse":
|
| 82 |
+
self.data = SparseTensorData(feats, coords, **kwargs)
|
| 83 |
+
elif BACKEND == "spconv":
|
| 84 |
+
spatial_shape = list(coords.max(0)[0] + 1)[1:]
|
| 85 |
+
self.data = SparseTensorData(
|
| 86 |
+
feats.reshape(feats.shape[0], -1),
|
| 87 |
+
coords,
|
| 88 |
+
spatial_shape,
|
| 89 |
+
shape[0],
|
| 90 |
+
**kwargs,
|
| 91 |
+
)
|
| 92 |
+
self.data._features = feats
|
| 93 |
+
elif method_id == 1:
|
| 94 |
+
data, shape, layout = args + (None,) * (3 - len(args))
|
| 95 |
+
if "data" in kwargs:
|
| 96 |
+
data = kwargs["data"]
|
| 97 |
+
del kwargs["data"]
|
| 98 |
+
if "shape" in kwargs:
|
| 99 |
+
shape = kwargs["shape"]
|
| 100 |
+
del kwargs["shape"]
|
| 101 |
+
if "layout" in kwargs:
|
| 102 |
+
layout = kwargs["layout"]
|
| 103 |
+
del kwargs["layout"]
|
| 104 |
+
|
| 105 |
+
self.data = data
|
| 106 |
+
if shape is None:
|
| 107 |
+
shape = self.__cal_shape(self.feats, self.coords)
|
| 108 |
+
if layout is None:
|
| 109 |
+
layout = self.__cal_layout(self.coords, shape[0])
|
| 110 |
+
|
| 111 |
+
self._shape = shape
|
| 112 |
+
self._layout = layout
|
| 113 |
+
self._scale = kwargs.get("scale", (1, 1, 1))
|
| 114 |
+
self._spatial_cache = kwargs.get("spatial_cache", {})
|
| 115 |
+
|
| 116 |
+
if DEBUG:
|
| 117 |
+
try:
|
| 118 |
+
assert (
|
| 119 |
+
self.feats.shape[0] == self.coords.shape[0]
|
| 120 |
+
), f"Invalid feats shape: {self.feats.shape}, coords shape: {self.coords.shape}"
|
| 121 |
+
assert self.shape == self.__cal_shape(
|
| 122 |
+
self.feats, self.coords
|
| 123 |
+
), f"Invalid shape: {self.shape}"
|
| 124 |
+
assert self.layout == self.__cal_layout(
|
| 125 |
+
self.coords, self.shape[0]
|
| 126 |
+
), f"Invalid layout: {self.layout}"
|
| 127 |
+
for i in range(self.shape[0]):
|
| 128 |
+
assert torch.all(
|
| 129 |
+
self.coords[self.layout[i], 0] == i
|
| 130 |
+
), f"The data of batch {i} is not contiguous"
|
| 131 |
+
except Exception as e:
|
| 132 |
+
print("Debugging information:")
|
| 133 |
+
print(f"- Shape: {self.shape}")
|
| 134 |
+
print(f"- Layout: {self.layout}")
|
| 135 |
+
print(f"- Scale: {self._scale}")
|
| 136 |
+
print(f"- Coords: {self.coords}")
|
| 137 |
+
raise e
|
| 138 |
+
|
| 139 |
+
def __cal_shape(self, feats, coords):
|
| 140 |
+
shape = []
|
| 141 |
+
shape.append(coords[:, 0].max().item() + 1)
|
| 142 |
+
shape.extend([*feats.shape[1:]])
|
| 143 |
+
return torch.Size(shape)
|
| 144 |
+
|
| 145 |
+
def __cal_layout(self, coords, batch_size):
|
| 146 |
+
seq_len = torch.bincount(coords[:, 0], minlength=batch_size)
|
| 147 |
+
offset = torch.cumsum(seq_len, dim=0)
|
| 148 |
+
# one host transfer instead of 2 .item() syncs per batch element (same slices)
|
| 149 |
+
starts, stops = torch.stack([offset - seq_len, offset]).tolist()
|
| 150 |
+
layout = [slice(a, b) for a, b in zip(starts, stops)]
|
| 151 |
+
return layout
|
| 152 |
+
|
| 153 |
+
@property
|
| 154 |
+
def shape(self) -> torch.Size:
|
| 155 |
+
return self._shape
|
| 156 |
+
|
| 157 |
+
def dim(self) -> int:
|
| 158 |
+
return len(self.shape)
|
| 159 |
+
|
| 160 |
+
@property
|
| 161 |
+
def layout(self) -> List[slice]:
|
| 162 |
+
return self._layout
|
| 163 |
+
|
| 164 |
+
@property
|
| 165 |
+
def feats(self) -> torch.Tensor:
|
| 166 |
+
if BACKEND == "torchsparse":
|
| 167 |
+
return self.data.F
|
| 168 |
+
elif BACKEND == "spconv":
|
| 169 |
+
return self.data.features
|
| 170 |
+
|
| 171 |
+
@feats.setter
|
| 172 |
+
def feats(self, value: torch.Tensor):
|
| 173 |
+
if BACKEND == "torchsparse":
|
| 174 |
+
self.data.F = value
|
| 175 |
+
elif BACKEND == "spconv":
|
| 176 |
+
self.data.features = value
|
| 177 |
+
|
| 178 |
+
@property
|
| 179 |
+
def coords(self) -> torch.Tensor:
|
| 180 |
+
if BACKEND == "torchsparse":
|
| 181 |
+
return self.data.C
|
| 182 |
+
elif BACKEND == "spconv":
|
| 183 |
+
return self.data.indices
|
| 184 |
+
|
| 185 |
+
@coords.setter
|
| 186 |
+
def coords(self, value: torch.Tensor):
|
| 187 |
+
if BACKEND == "torchsparse":
|
| 188 |
+
self.data.C = value
|
| 189 |
+
elif BACKEND == "spconv":
|
| 190 |
+
self.data.indices = value
|
| 191 |
+
|
| 192 |
+
@property
|
| 193 |
+
def dtype(self):
|
| 194 |
+
return self.feats.dtype
|
| 195 |
+
|
| 196 |
+
@property
|
| 197 |
+
def device(self):
|
| 198 |
+
return self.feats.device
|
| 199 |
+
|
| 200 |
+
@overload
|
| 201 |
+
def to(self, dtype: torch.dtype) -> "SparseTensor": ...
|
| 202 |
+
|
| 203 |
+
@overload
|
| 204 |
+
def to(
|
| 205 |
+
self,
|
| 206 |
+
device: Optional[Union[str, torch.device]] = None,
|
| 207 |
+
dtype: Optional[torch.dtype] = None,
|
| 208 |
+
) -> "SparseTensor": ...
|
| 209 |
+
|
| 210 |
+
def to(self, *args, **kwargs) -> "SparseTensor":
|
| 211 |
+
device = None
|
| 212 |
+
dtype = None
|
| 213 |
+
if len(args) == 2:
|
| 214 |
+
device, dtype = args
|
| 215 |
+
elif len(args) == 1:
|
| 216 |
+
if isinstance(args[0], torch.dtype):
|
| 217 |
+
dtype = args[0]
|
| 218 |
+
else:
|
| 219 |
+
device = args[0]
|
| 220 |
+
if "dtype" in kwargs:
|
| 221 |
+
assert dtype is None, "to() received multiple values for argument 'dtype'"
|
| 222 |
+
dtype = kwargs["dtype"]
|
| 223 |
+
if "device" in kwargs:
|
| 224 |
+
assert device is None, "to() received multiple values for argument 'device'"
|
| 225 |
+
device = kwargs["device"]
|
| 226 |
+
|
| 227 |
+
new_feats = self.feats.to(device=device, dtype=dtype)
|
| 228 |
+
new_coords = self.coords.to(device=device)
|
| 229 |
+
return self.replace(new_feats, new_coords)
|
| 230 |
+
|
| 231 |
+
def type(self, dtype):
|
| 232 |
+
new_feats = self.feats.type(dtype)
|
| 233 |
+
return self.replace(new_feats)
|
| 234 |
+
|
| 235 |
+
def cpu(self) -> "SparseTensor":
|
| 236 |
+
new_feats = self.feats.cpu()
|
| 237 |
+
new_coords = self.coords.cpu()
|
| 238 |
+
return self.replace(new_feats, new_coords)
|
| 239 |
+
|
| 240 |
+
def cuda(self) -> "SparseTensor":
|
| 241 |
+
new_feats = self.feats.cuda()
|
| 242 |
+
new_coords = self.coords.cuda()
|
| 243 |
+
return self.replace(new_feats, new_coords)
|
| 244 |
+
|
| 245 |
+
def half(self) -> "SparseTensor":
|
| 246 |
+
new_feats = self.feats.half()
|
| 247 |
+
return self.replace(new_feats)
|
| 248 |
+
|
| 249 |
+
def float(self) -> "SparseTensor":
|
| 250 |
+
new_feats = self.feats.float()
|
| 251 |
+
return self.replace(new_feats)
|
| 252 |
+
|
| 253 |
+
def detach(self) -> "SparseTensor":
|
| 254 |
+
new_coords = self.coords.detach()
|
| 255 |
+
new_feats = self.feats.detach()
|
| 256 |
+
return self.replace(new_feats, new_coords)
|
| 257 |
+
|
| 258 |
+
def dense(self) -> torch.Tensor:
|
| 259 |
+
if BACKEND == "torchsparse":
|
| 260 |
+
return self.data.dense()
|
| 261 |
+
elif BACKEND == "spconv":
|
| 262 |
+
return self.data.dense()
|
| 263 |
+
|
| 264 |
+
def reshape(self, *shape) -> "SparseTensor":
|
| 265 |
+
new_feats = self.feats.reshape(self.feats.shape[0], *shape)
|
| 266 |
+
return self.replace(new_feats)
|
| 267 |
+
|
| 268 |
+
def unbind(self, dim: int) -> List["SparseTensor"]:
|
| 269 |
+
return sparse_unbind(self, dim)
|
| 270 |
+
|
| 271 |
+
def replace(
|
| 272 |
+
self, feats: torch.Tensor, coords: Optional[torch.Tensor] = None
|
| 273 |
+
) -> "SparseTensor":
|
| 274 |
+
new_shape = [self.shape[0]]
|
| 275 |
+
new_shape.extend(feats.shape[1:])
|
| 276 |
+
if BACKEND == "torchsparse":
|
| 277 |
+
new_data = SparseTensorData(
|
| 278 |
+
feats=feats,
|
| 279 |
+
coords=self.data.coords if coords is None else coords,
|
| 280 |
+
stride=self.data.stride,
|
| 281 |
+
spatial_range=self.data.spatial_range,
|
| 282 |
+
)
|
| 283 |
+
new_data._caches = self.data._caches
|
| 284 |
+
elif BACKEND == "spconv":
|
| 285 |
+
new_data = SparseTensorData(
|
| 286 |
+
self.data.features.reshape(self.data.features.shape[0], -1),
|
| 287 |
+
self.data.indices,
|
| 288 |
+
self.data.spatial_shape,
|
| 289 |
+
self.data.batch_size,
|
| 290 |
+
self.data.grid,
|
| 291 |
+
self.data.voxel_num,
|
| 292 |
+
self.data.indice_dict,
|
| 293 |
+
)
|
| 294 |
+
new_data._features = feats
|
| 295 |
+
new_data.benchmark = self.data.benchmark
|
| 296 |
+
new_data.benchmark_record = self.data.benchmark_record
|
| 297 |
+
new_data.thrust_allocator = self.data.thrust_allocator
|
| 298 |
+
new_data._timer = self.data._timer
|
| 299 |
+
new_data.force_algo = self.data.force_algo
|
| 300 |
+
new_data.int8_scale = self.data.int8_scale
|
| 301 |
+
if coords is not None:
|
| 302 |
+
new_data.indices = coords
|
| 303 |
+
new_tensor = SparseTensor(
|
| 304 |
+
new_data,
|
| 305 |
+
shape=torch.Size(new_shape),
|
| 306 |
+
layout=self.layout,
|
| 307 |
+
scale=self._scale,
|
| 308 |
+
spatial_cache=self._spatial_cache,
|
| 309 |
+
)
|
| 310 |
+
return new_tensor
|
| 311 |
+
|
| 312 |
+
@staticmethod
|
| 313 |
+
def full(aabb, dim, value, dtype=torch.float32, device=None) -> "SparseTensor":
|
| 314 |
+
N, C = dim
|
| 315 |
+
x = torch.arange(aabb[0], aabb[3] + 1)
|
| 316 |
+
y = torch.arange(aabb[1], aabb[4] + 1)
|
| 317 |
+
z = torch.arange(aabb[2], aabb[5] + 1)
|
| 318 |
+
coords = torch.stack(torch.meshgrid(x, y, z, indexing="ij"), dim=-1).reshape(
|
| 319 |
+
-1, 3
|
| 320 |
+
)
|
| 321 |
+
coords = torch.cat(
|
| 322 |
+
[
|
| 323 |
+
torch.arange(N).view(-1, 1).repeat(1, coords.shape[0]).view(-1, 1),
|
| 324 |
+
coords.repeat(N, 1),
|
| 325 |
+
],
|
| 326 |
+
dim=1,
|
| 327 |
+
).to(dtype=torch.int32, device=device)
|
| 328 |
+
feats = torch.full((coords.shape[0], C), value, dtype=dtype, device=device)
|
| 329 |
+
return SparseTensor(feats=feats, coords=coords)
|
| 330 |
+
|
| 331 |
+
def __merge_sparse_cache(self, other: "SparseTensor") -> dict:
|
| 332 |
+
new_cache = {}
|
| 333 |
+
for k in set(
|
| 334 |
+
list(self._spatial_cache.keys()) + list(other._spatial_cache.keys())
|
| 335 |
+
):
|
| 336 |
+
if k in self._spatial_cache:
|
| 337 |
+
new_cache[k] = self._spatial_cache[k]
|
| 338 |
+
if k in other._spatial_cache:
|
| 339 |
+
if k not in new_cache:
|
| 340 |
+
new_cache[k] = other._spatial_cache[k]
|
| 341 |
+
else:
|
| 342 |
+
new_cache[k].update(other._spatial_cache[k])
|
| 343 |
+
return new_cache
|
| 344 |
+
|
| 345 |
+
def __neg__(self) -> "SparseTensor":
|
| 346 |
+
return self.replace(-self.feats)
|
| 347 |
+
|
| 348 |
+
def __elemwise__(
|
| 349 |
+
self, other: Union[torch.Tensor, "SparseTensor"], op: callable
|
| 350 |
+
) -> "SparseTensor":
|
| 351 |
+
if isinstance(other, torch.Tensor):
|
| 352 |
+
try:
|
| 353 |
+
other = torch.broadcast_to(other, self.shape)
|
| 354 |
+
other = sparse_batch_broadcast(self, other)
|
| 355 |
+
except:
|
| 356 |
+
pass
|
| 357 |
+
if isinstance(other, SparseTensor):
|
| 358 |
+
other = other.feats
|
| 359 |
+
new_feats = op(self.feats, other)
|
| 360 |
+
new_tensor = self.replace(new_feats)
|
| 361 |
+
if isinstance(other, SparseTensor):
|
| 362 |
+
new_tensor._spatial_cache = self.__merge_sparse_cache(other)
|
| 363 |
+
return new_tensor
|
| 364 |
+
|
| 365 |
+
def __add__(
|
| 366 |
+
self, other: Union[torch.Tensor, "SparseTensor", float]
|
| 367 |
+
) -> "SparseTensor":
|
| 368 |
+
return self.__elemwise__(other, torch.add)
|
| 369 |
+
|
| 370 |
+
def __radd__(
|
| 371 |
+
self, other: Union[torch.Tensor, "SparseTensor", float]
|
| 372 |
+
) -> "SparseTensor":
|
| 373 |
+
return self.__elemwise__(other, torch.add)
|
| 374 |
+
|
| 375 |
+
def __sub__(
|
| 376 |
+
self, other: Union[torch.Tensor, "SparseTensor", float]
|
| 377 |
+
) -> "SparseTensor":
|
| 378 |
+
return self.__elemwise__(other, torch.sub)
|
| 379 |
+
|
| 380 |
+
def __rsub__(
|
| 381 |
+
self, other: Union[torch.Tensor, "SparseTensor", float]
|
| 382 |
+
) -> "SparseTensor":
|
| 383 |
+
return self.__elemwise__(other, lambda x, y: torch.sub(y, x))
|
| 384 |
+
|
| 385 |
+
def __mul__(
|
| 386 |
+
self, other: Union[torch.Tensor, "SparseTensor", float]
|
| 387 |
+
) -> "SparseTensor":
|
| 388 |
+
return self.__elemwise__(other, torch.mul)
|
| 389 |
+
|
| 390 |
+
def __rmul__(
|
| 391 |
+
self, other: Union[torch.Tensor, "SparseTensor", float]
|
| 392 |
+
) -> "SparseTensor":
|
| 393 |
+
return self.__elemwise__(other, torch.mul)
|
| 394 |
+
|
| 395 |
+
def __truediv__(
|
| 396 |
+
self, other: Union[torch.Tensor, "SparseTensor", float]
|
| 397 |
+
) -> "SparseTensor":
|
| 398 |
+
return self.__elemwise__(other, torch.div)
|
| 399 |
+
|
| 400 |
+
def __rtruediv__(
|
| 401 |
+
self, other: Union[torch.Tensor, "SparseTensor", float]
|
| 402 |
+
) -> "SparseTensor":
|
| 403 |
+
return self.__elemwise__(other, lambda x, y: torch.div(y, x))
|
| 404 |
+
|
| 405 |
+
def __getitem__(self, idx):
|
| 406 |
+
if isinstance(idx, int):
|
| 407 |
+
idx = [idx]
|
| 408 |
+
elif isinstance(idx, slice):
|
| 409 |
+
idx = range(*idx.indices(self.shape[0]))
|
| 410 |
+
elif isinstance(idx, torch.Tensor):
|
| 411 |
+
if idx.dtype == torch.bool:
|
| 412 |
+
assert idx.shape == (
|
| 413 |
+
self.shape[0],
|
| 414 |
+
), f"Invalid index shape: {idx.shape}"
|
| 415 |
+
idx = idx.nonzero().squeeze(1)
|
| 416 |
+
elif idx.dtype in [torch.int32, torch.int64]:
|
| 417 |
+
assert len(idx.shape) == 1, f"Invalid index shape: {idx.shape}"
|
| 418 |
+
else:
|
| 419 |
+
raise ValueError(f"Unknown index type: {idx.dtype}")
|
| 420 |
+
else:
|
| 421 |
+
raise ValueError(f"Unknown index type: {type(idx)}")
|
| 422 |
+
|
| 423 |
+
coords = []
|
| 424 |
+
feats = []
|
| 425 |
+
for new_idx, old_idx in enumerate(idx):
|
| 426 |
+
coords.append(self.coords[self.layout[old_idx]].clone())
|
| 427 |
+
coords[-1][:, 0] = new_idx
|
| 428 |
+
feats.append(self.feats[self.layout[old_idx]])
|
| 429 |
+
coords = torch.cat(coords, dim=0).contiguous()
|
| 430 |
+
feats = torch.cat(feats, dim=0).contiguous()
|
| 431 |
+
return SparseTensor(feats=feats, coords=coords)
|
| 432 |
+
|
| 433 |
+
def register_spatial_cache(self, key, value) -> None:
|
| 434 |
+
"""
|
| 435 |
+
Register a spatial cache.
|
| 436 |
+
The spatial cache can be any thing you want to cache.
|
| 437 |
+
The registery and retrieval of the cache is based on current scale.
|
| 438 |
+
"""
|
| 439 |
+
scale_key = str(self._scale)
|
| 440 |
+
if scale_key not in self._spatial_cache:
|
| 441 |
+
self._spatial_cache[scale_key] = {}
|
| 442 |
+
self._spatial_cache[scale_key][key] = value
|
| 443 |
+
|
| 444 |
+
def get_spatial_cache(self, key=None):
|
| 445 |
+
"""
|
| 446 |
+
Get a spatial cache.
|
| 447 |
+
"""
|
| 448 |
+
scale_key = str(self._scale)
|
| 449 |
+
cur_scale_cache = self._spatial_cache.get(scale_key, {})
|
| 450 |
+
if key is None:
|
| 451 |
+
return cur_scale_cache
|
| 452 |
+
return cur_scale_cache.get(key, None)
|
| 453 |
+
|
| 454 |
+
|
| 455 |
+
class _BatchBroadcast(torch.autograd.Function):
|
| 456 |
+
@staticmethod
|
| 457 |
+
def forward(ctx, other, idx, out_dtype, out_shape):
|
| 458 |
+
ctx.save_for_backward(idx)
|
| 459 |
+
ctx.other_shape = other.shape
|
| 460 |
+
ctx.other_dtype = other.dtype
|
| 461 |
+
rows = other[idx] # [N, *other.shape[1:]]
|
| 462 |
+
# broadcast each row's value like the loop's `broadcasted[rows] = other[k]`
|
| 463 |
+
# (right-aligned): insert singleton dims after the batch dim, then expand
|
| 464 |
+
extra = len(out_shape) - rows.dim()
|
| 465 |
+
if extra > 0:
|
| 466 |
+
rows = rows.reshape(rows.shape[0], *([1] * extra), *rows.shape[1:])
|
| 467 |
+
return rows.to(out_dtype).expand(out_shape).contiguous()
|
| 468 |
+
|
| 469 |
+
@staticmethod
|
| 470 |
+
def backward(ctx, grad):
|
| 471 |
+
(idx,) = ctx.saved_tensors
|
| 472 |
+
acc_dt = torch.float64 if ctx.other_dtype == torch.float64 else torch.float32
|
| 473 |
+
g = grad.to(ctx.other_dtype) # per-row cast (as the loop)
|
| 474 |
+
# sum over any broadcast (non-batch) dims, like ExpandBackward on other[k]
|
| 475 |
+
tgt = ctx.other_shape[1:]
|
| 476 |
+
while g.dim() - 1 > len(tgt):
|
| 477 |
+
g = g.sum(1, dtype=acc_dt).to(ctx.other_dtype)
|
| 478 |
+
for d, sz in enumerate(tgt):
|
| 479 |
+
if sz == 1 and g.shape[d + 1] != 1:
|
| 480 |
+
g = g.sum(d + 1, keepdim=True, dtype=acc_dt).to(ctx.other_dtype)
|
| 481 |
+
acc = torch.zeros((ctx.other_shape[0],) + tuple(g.shape[1:]), dtype=acc_dt,
|
| 482 |
+
device=g.device)
|
| 483 |
+
acc.index_add_(0, idx, g.to(acc_dt))
|
| 484 |
+
return acc.to(ctx.other_dtype), None, None, None
|
| 485 |
+
|
| 486 |
+
|
| 487 |
+
import os as _os
|
| 488 |
+
_BROADCAST_LOOP = _os.environ.get("MVSAM3D_BROADCAST_LOOP", "0") == "1" # 1 = original loop
|
| 489 |
+
|
| 490 |
+
|
| 491 |
+
def sparse_batch_broadcast(input: SparseTensor, other: torch.Tensor) -> torch.Tensor:
|
| 492 |
+
"""
|
| 493 |
+
Broadcast a 1D tensor to a sparse tensor along the batch dimension then perform an operation.
|
| 494 |
+
|
| 495 |
+
Args:
|
| 496 |
+
input (torch.Tensor): 1D tensor to broadcast.
|
| 497 |
+
target (SparseTensor): Sparse tensor to broadcast to.
|
| 498 |
+
op (callable): Operation to perform after broadcasting. Defaults to torch.add.
|
| 499 |
+
"""
|
| 500 |
+
coords, feats = input.coords, input.feats
|
| 501 |
+
if _BROADCAST_LOOP:
|
| 502 |
+
broadcasted = torch.zeros_like(feats)
|
| 503 |
+
for k in range(input.shape[0]):
|
| 504 |
+
broadcasted[input.layout[k]] = other[k]
|
| 505 |
+
return broadcasted
|
| 506 |
+
# Same forward values as the per-batch slice-assignment loop above, as one
|
| 507 |
+
# gather: row r gets other[batch(r)] cast to feats.dtype. Backward: each
|
| 508 |
+
# row's gradient is cast to other.dtype, then summed per batch element with
|
| 509 |
+
# index_add_ (fp32 accumulation, one final rounding to other.dtype). The
|
| 510 |
+
# summation ORDER differs from the loop's reduction (CUDA index_add_ uses
|
| 511 |
+
# atomics), so the backward matches the loop to fp32 rounding, not bitwise.
|
| 512 |
+
idx = coords[:, 0].long()
|
| 513 |
+
return _BatchBroadcast.apply(other, idx, feats.dtype, tuple(feats.shape))
|
| 514 |
+
|
| 515 |
+
|
| 516 |
+
def sparse_batch_op(
|
| 517 |
+
input: SparseTensor, other: torch.Tensor, op: callable = torch.add
|
| 518 |
+
) -> SparseTensor:
|
| 519 |
+
"""
|
| 520 |
+
Broadcast a 1D tensor to a sparse tensor along the batch dimension then perform an operation.
|
| 521 |
+
|
| 522 |
+
Args:
|
| 523 |
+
input (torch.Tensor): 1D tensor to broadcast.
|
| 524 |
+
target (SparseTensor): Sparse tensor to broadcast to.
|
| 525 |
+
op (callable): Operation to perform after broadcasting. Defaults to torch.add.
|
| 526 |
+
"""
|
| 527 |
+
return input.replace(op(input.feats, sparse_batch_broadcast(input, other)))
|
| 528 |
+
|
| 529 |
+
|
| 530 |
+
def sparse_cat(inputs: List[SparseTensor], dim: int = 0) -> SparseTensor:
|
| 531 |
+
"""
|
| 532 |
+
Concatenate a list of sparse tensors.
|
| 533 |
+
|
| 534 |
+
Args:
|
| 535 |
+
inputs (List[SparseTensor]): List of sparse tensors to concatenate.
|
| 536 |
+
"""
|
| 537 |
+
if dim == 0:
|
| 538 |
+
start = 0
|
| 539 |
+
coords = []
|
| 540 |
+
for input in inputs:
|
| 541 |
+
coords.append(input.coords.clone())
|
| 542 |
+
coords[-1][:, 0] += start
|
| 543 |
+
start += input.shape[0]
|
| 544 |
+
coords = torch.cat(coords, dim=0)
|
| 545 |
+
feats = torch.cat([input.feats for input in inputs], dim=0)
|
| 546 |
+
output = SparseTensor(
|
| 547 |
+
coords=coords,
|
| 548 |
+
feats=feats,
|
| 549 |
+
)
|
| 550 |
+
else:
|
| 551 |
+
feats = torch.cat([input.feats for input in inputs], dim=dim)
|
| 552 |
+
output = inputs[0].replace(feats)
|
| 553 |
+
|
| 554 |
+
return output
|
| 555 |
+
|
| 556 |
+
|
| 557 |
+
def sparse_unbind(input: SparseTensor, dim: int) -> List[SparseTensor]:
|
| 558 |
+
"""
|
| 559 |
+
Unbind a sparse tensor along a dimension.
|
| 560 |
+
|
| 561 |
+
Args:
|
| 562 |
+
input (SparseTensor): Sparse tensor to unbind.
|
| 563 |
+
dim (int): Dimension to unbind.
|
| 564 |
+
"""
|
| 565 |
+
if dim == 0:
|
| 566 |
+
return [input[i] for i in range(input.shape[0])]
|
| 567 |
+
else:
|
| 568 |
+
feats = input.feats.unbind(dim)
|
| 569 |
+
return [input.replace(f) for f in feats]
|
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/conv/__init__.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
from .. import BACKEND
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
SPCONV_ALGO = "auto" # 'auto', 'implicit_gemm', 'native'
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def __from_env():
|
| 9 |
+
import os
|
| 10 |
+
|
| 11 |
+
global SPCONV_ALGO
|
| 12 |
+
env_spconv_algo = os.environ.get("SPCONV_ALGO")
|
| 13 |
+
if env_spconv_algo is not None and env_spconv_algo in [
|
| 14 |
+
"auto",
|
| 15 |
+
"implicit_gemm",
|
| 16 |
+
"native",
|
| 17 |
+
]:
|
| 18 |
+
SPCONV_ALGO = env_spconv_algo
|
| 19 |
+
print(f"[SPARSE][CONV] spconv algo: {SPCONV_ALGO}")
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
__from_env()
|
| 23 |
+
|
| 24 |
+
if BACKEND == "torchsparse":
|
| 25 |
+
from .conv_torchsparse import *
|
| 26 |
+
elif BACKEND == "spconv":
|
| 27 |
+
from .conv_spconv import *
|
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/conv/conv_spconv.py
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn as nn
|
| 4 |
+
from .. import SparseTensor
|
| 5 |
+
from .. import DEBUG
|
| 6 |
+
from . import SPCONV_ALGO
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class SparseConv3d(nn.Module):
|
| 10 |
+
def __init__(
|
| 11 |
+
self,
|
| 12 |
+
in_channels,
|
| 13 |
+
out_channels,
|
| 14 |
+
kernel_size,
|
| 15 |
+
stride=1,
|
| 16 |
+
dilation=1,
|
| 17 |
+
padding=None,
|
| 18 |
+
bias=True,
|
| 19 |
+
indice_key=None,
|
| 20 |
+
):
|
| 21 |
+
super(SparseConv3d, self).__init__()
|
| 22 |
+
if "spconv" not in globals():
|
| 23 |
+
import spconv.pytorch as spconv
|
| 24 |
+
algo = None
|
| 25 |
+
if SPCONV_ALGO == "native":
|
| 26 |
+
algo = spconv.ConvAlgo.Native
|
| 27 |
+
elif SPCONV_ALGO == "implicit_gemm":
|
| 28 |
+
algo = spconv.ConvAlgo.MaskImplicitGemm
|
| 29 |
+
if stride == 1 and (padding is None):
|
| 30 |
+
self.conv = spconv.SubMConv3d(
|
| 31 |
+
in_channels,
|
| 32 |
+
out_channels,
|
| 33 |
+
kernel_size,
|
| 34 |
+
dilation=dilation,
|
| 35 |
+
bias=bias,
|
| 36 |
+
indice_key=indice_key,
|
| 37 |
+
algo=algo,
|
| 38 |
+
)
|
| 39 |
+
else:
|
| 40 |
+
self.conv = spconv.SparseConv3d(
|
| 41 |
+
in_channels,
|
| 42 |
+
out_channels,
|
| 43 |
+
kernel_size,
|
| 44 |
+
stride=stride,
|
| 45 |
+
dilation=dilation,
|
| 46 |
+
padding=padding,
|
| 47 |
+
bias=bias,
|
| 48 |
+
indice_key=indice_key,
|
| 49 |
+
algo=algo,
|
| 50 |
+
)
|
| 51 |
+
self.stride = (
|
| 52 |
+
tuple(stride)
|
| 53 |
+
if isinstance(stride, (list, tuple))
|
| 54 |
+
else (stride, stride, stride)
|
| 55 |
+
)
|
| 56 |
+
self.padding = padding
|
| 57 |
+
|
| 58 |
+
def forward(self, x: SparseTensor) -> SparseTensor:
|
| 59 |
+
spatial_changed = any(s != 1 for s in self.stride) or (self.padding is not None)
|
| 60 |
+
new_data = self.conv(x.data)
|
| 61 |
+
new_shape = [x.shape[0], self.conv.out_channels]
|
| 62 |
+
new_layout = None if spatial_changed else x.layout
|
| 63 |
+
|
| 64 |
+
if spatial_changed and (x.shape[0] != 1):
|
| 65 |
+
# spconv was non-1 stride will break the contiguous of the output tensor, sort by the coords
|
| 66 |
+
fwd = new_data.indices[:, 0].argsort()
|
| 67 |
+
bwd = torch.zeros_like(fwd).scatter_(
|
| 68 |
+
0, fwd, torch.arange(fwd.shape[0], device=fwd.device)
|
| 69 |
+
)
|
| 70 |
+
sorted_feats = new_data.features[fwd]
|
| 71 |
+
sorted_coords = new_data.indices[fwd]
|
| 72 |
+
unsorted_data = new_data
|
| 73 |
+
new_data = spconv.SparseConvTensor(sorted_feats, sorted_coords, unsorted_data.spatial_shape, unsorted_data.batch_size) # type: ignore
|
| 74 |
+
|
| 75 |
+
out = SparseTensor(
|
| 76 |
+
new_data,
|
| 77 |
+
shape=torch.Size(new_shape),
|
| 78 |
+
layout=new_layout,
|
| 79 |
+
scale=tuple([s * stride for s, stride in zip(x._scale, self.stride)]),
|
| 80 |
+
spatial_cache=x._spatial_cache,
|
| 81 |
+
)
|
| 82 |
+
|
| 83 |
+
if spatial_changed and (x.shape[0] != 1):
|
| 84 |
+
out.register_spatial_cache(
|
| 85 |
+
f"conv_{self.stride}_unsorted_data", unsorted_data
|
| 86 |
+
)
|
| 87 |
+
out.register_spatial_cache(f"conv_{self.stride}_sort_bwd", bwd)
|
| 88 |
+
|
| 89 |
+
return out
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
class SparseInverseConv3d(nn.Module):
|
| 93 |
+
def __init__(
|
| 94 |
+
self,
|
| 95 |
+
in_channels,
|
| 96 |
+
out_channels,
|
| 97 |
+
kernel_size,
|
| 98 |
+
stride=1,
|
| 99 |
+
dilation=1,
|
| 100 |
+
bias=True,
|
| 101 |
+
indice_key=None,
|
| 102 |
+
):
|
| 103 |
+
super(SparseInverseConv3d, self).__init__()
|
| 104 |
+
if "spconv" not in globals():
|
| 105 |
+
import spconv.pytorch as spconv
|
| 106 |
+
self.conv = spconv.SparseInverseConv3d(
|
| 107 |
+
in_channels, out_channels, kernel_size, bias=bias, indice_key=indice_key
|
| 108 |
+
)
|
| 109 |
+
self.stride = (
|
| 110 |
+
tuple(stride)
|
| 111 |
+
if isinstance(stride, (list, tuple))
|
| 112 |
+
else (stride, stride, stride)
|
| 113 |
+
)
|
| 114 |
+
|
| 115 |
+
def forward(self, x: SparseTensor) -> SparseTensor:
|
| 116 |
+
spatial_changed = any(s != 1 for s in self.stride)
|
| 117 |
+
if spatial_changed:
|
| 118 |
+
# recover the original spconv order
|
| 119 |
+
data = x.get_spatial_cache(f"conv_{self.stride}_unsorted_data")
|
| 120 |
+
bwd = x.get_spatial_cache(f"conv_{self.stride}_sort_bwd")
|
| 121 |
+
data = data.replace_feature(x.feats[bwd])
|
| 122 |
+
if DEBUG:
|
| 123 |
+
assert torch.equal(
|
| 124 |
+
data.indices, x.coords[bwd]
|
| 125 |
+
), "Recover the original order failed"
|
| 126 |
+
else:
|
| 127 |
+
data = x.data
|
| 128 |
+
|
| 129 |
+
new_data = self.conv(data)
|
| 130 |
+
new_shape = [x.shape[0], self.conv.out_channels]
|
| 131 |
+
new_layout = None if spatial_changed else x.layout
|
| 132 |
+
out = SparseTensor(
|
| 133 |
+
new_data,
|
| 134 |
+
shape=torch.Size(new_shape),
|
| 135 |
+
layout=new_layout,
|
| 136 |
+
scale=tuple([s // stride for s, stride in zip(x._scale, self.stride)]),
|
| 137 |
+
spatial_cache=x._spatial_cache,
|
| 138 |
+
)
|
| 139 |
+
return out
|
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/linear.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn as nn
|
| 4 |
+
from . import SparseTensor
|
| 5 |
+
|
| 6 |
+
__all__ = ["SparseLinear"]
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class SparseLinear(nn.Linear):
|
| 10 |
+
def __init__(self, in_features, out_features, bias=True):
|
| 11 |
+
super(SparseLinear, self).__init__(in_features, out_features, bias)
|
| 12 |
+
|
| 13 |
+
def forward(self, input: SparseTensor) -> SparseTensor:
|
| 14 |
+
return input.replace(super().forward(input.feats))
|
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/nonlinearity.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn as nn
|
| 4 |
+
from . import SparseTensor
|
| 5 |
+
|
| 6 |
+
__all__ = ["SparseReLU", "SparseSiLU", "SparseGELU", "SparseActivation"]
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class SparseReLU(nn.ReLU):
|
| 10 |
+
def forward(self, input: SparseTensor) -> SparseTensor:
|
| 11 |
+
return input.replace(super().forward(input.feats))
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class SparseSiLU(nn.SiLU):
|
| 15 |
+
def forward(self, input: SparseTensor) -> SparseTensor:
|
| 16 |
+
return input.replace(super().forward(input.feats))
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class SparseGELU(nn.GELU):
|
| 20 |
+
def forward(self, input: SparseTensor) -> SparseTensor:
|
| 21 |
+
return input.replace(super().forward(input.feats))
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class SparseActivation(nn.Module):
|
| 25 |
+
def __init__(self, activation: nn.Module):
|
| 26 |
+
super().__init__()
|
| 27 |
+
self.activation = activation
|
| 28 |
+
|
| 29 |
+
def forward(self, input: SparseTensor) -> SparseTensor:
|
| 30 |
+
return input.replace(self.activation(input.feats))
|
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/norm.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn as nn
|
| 4 |
+
from . import SparseTensor
|
| 5 |
+
from . import DEBUG
|
| 6 |
+
|
| 7 |
+
__all__ = [
|
| 8 |
+
"SparseGroupNorm",
|
| 9 |
+
"SparseLayerNorm",
|
| 10 |
+
"SparseGroupNorm32",
|
| 11 |
+
"SparseLayerNorm32",
|
| 12 |
+
]
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class SparseGroupNorm(nn.GroupNorm):
|
| 16 |
+
def __init__(self, num_groups, num_channels, eps=1e-5, affine=True):
|
| 17 |
+
super(SparseGroupNorm, self).__init__(num_groups, num_channels, eps, affine)
|
| 18 |
+
|
| 19 |
+
def forward(self, input: SparseTensor) -> SparseTensor:
|
| 20 |
+
nfeats = torch.zeros_like(input.feats)
|
| 21 |
+
for k in range(input.shape[0]):
|
| 22 |
+
if DEBUG:
|
| 23 |
+
assert (
|
| 24 |
+
input.coords[input.layout[k], 0] == k
|
| 25 |
+
).all(), f"SparseGroupNorm: batch index mismatch"
|
| 26 |
+
bfeats = input.feats[input.layout[k]]
|
| 27 |
+
bfeats = bfeats.permute(1, 0).reshape(1, input.shape[1], -1)
|
| 28 |
+
bfeats = super().forward(bfeats)
|
| 29 |
+
bfeats = bfeats.reshape(input.shape[1], -1).permute(1, 0)
|
| 30 |
+
nfeats[input.layout[k]] = bfeats
|
| 31 |
+
return input.replace(nfeats)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
class SparseLayerNorm(nn.LayerNorm):
|
| 35 |
+
def __init__(self, normalized_shape, eps=1e-5, elementwise_affine=True):
|
| 36 |
+
super(SparseLayerNorm, self).__init__(normalized_shape, eps, elementwise_affine)
|
| 37 |
+
|
| 38 |
+
def forward(self, input: SparseTensor) -> SparseTensor:
|
| 39 |
+
nfeats = torch.zeros_like(input.feats)
|
| 40 |
+
for k in range(input.shape[0]):
|
| 41 |
+
bfeats = input.feats[input.layout[k]]
|
| 42 |
+
bfeats = bfeats.permute(1, 0).reshape(1, input.shape[1], -1)
|
| 43 |
+
bfeats = super().forward(bfeats)
|
| 44 |
+
bfeats = bfeats.reshape(input.shape[1], -1).permute(1, 0)
|
| 45 |
+
nfeats[input.layout[k]] = bfeats
|
| 46 |
+
return input.replace(nfeats)
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
class SparseGroupNorm32(SparseGroupNorm):
|
| 50 |
+
"""
|
| 51 |
+
A GroupNorm layer that converts to float32 before the forward pass.
|
| 52 |
+
"""
|
| 53 |
+
|
| 54 |
+
def forward(self, x: SparseTensor) -> SparseTensor:
|
| 55 |
+
return super().forward(x.float()).type(x.dtype)
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
class SparseLayerNorm32(SparseLayerNorm):
|
| 59 |
+
"""
|
| 60 |
+
A LayerNorm layer that converts to float32 before the forward pass.
|
| 61 |
+
"""
|
| 62 |
+
|
| 63 |
+
def forward(self, x: SparseTensor) -> SparseTensor:
|
| 64 |
+
return super().forward(x.float()).type(x.dtype)
|
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/spatial.py
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
from typing import *
|
| 3 |
+
import torch
|
| 4 |
+
import torch.nn as nn
|
| 5 |
+
from . import SparseTensor
|
| 6 |
+
|
| 7 |
+
__all__ = ["SparseDownsample", "SparseUpsample", "SparseSubdivide"]
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class SparseDownsample(nn.Module):
|
| 11 |
+
"""
|
| 12 |
+
Downsample a sparse tensor by a factor of `factor`.
|
| 13 |
+
Implemented as average pooling.
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
def __init__(self, factor: Union[int, Tuple[int, ...], List[int]]):
|
| 17 |
+
super(SparseDownsample, self).__init__()
|
| 18 |
+
self.factor = tuple(factor) if isinstance(factor, (list, tuple)) else factor
|
| 19 |
+
|
| 20 |
+
def forward(self, input: SparseTensor) -> SparseTensor:
|
| 21 |
+
DIM = input.coords.shape[-1] - 1
|
| 22 |
+
factor = self.factor if isinstance(self.factor, tuple) else (self.factor,) * DIM
|
| 23 |
+
assert DIM == len(
|
| 24 |
+
factor
|
| 25 |
+
), "Input coordinates must have the same dimension as the downsample factor."
|
| 26 |
+
|
| 27 |
+
coord = list(input.coords.unbind(dim=-1))
|
| 28 |
+
for i, f in enumerate(factor):
|
| 29 |
+
coord[i + 1] = coord[i + 1] // f
|
| 30 |
+
|
| 31 |
+
MAX = [coord[i + 1].max().item() + 1 for i in range(DIM)]
|
| 32 |
+
OFFSET = torch.cumprod(torch.tensor(MAX[::-1]), 0).tolist()[::-1] + [1]
|
| 33 |
+
code = sum([c * o for c, o in zip(coord, OFFSET)])
|
| 34 |
+
code, idx = code.unique(return_inverse=True)
|
| 35 |
+
|
| 36 |
+
new_feats = torch.scatter_reduce(
|
| 37 |
+
torch.zeros(
|
| 38 |
+
code.shape[0],
|
| 39 |
+
input.feats.shape[1],
|
| 40 |
+
device=input.feats.device,
|
| 41 |
+
dtype=input.feats.dtype,
|
| 42 |
+
),
|
| 43 |
+
dim=0,
|
| 44 |
+
index=idx.unsqueeze(1).expand(-1, input.feats.shape[1]),
|
| 45 |
+
src=input.feats,
|
| 46 |
+
reduce="mean",
|
| 47 |
+
)
|
| 48 |
+
new_coords = torch.stack(
|
| 49 |
+
[code // OFFSET[0]]
|
| 50 |
+
+ [(code // OFFSET[i + 1]) % MAX[i] for i in range(DIM)],
|
| 51 |
+
dim=-1,
|
| 52 |
+
)
|
| 53 |
+
out = SparseTensor(
|
| 54 |
+
new_feats,
|
| 55 |
+
new_coords,
|
| 56 |
+
input.shape,
|
| 57 |
+
)
|
| 58 |
+
out._scale = tuple([s // f for s, f in zip(input._scale, factor)])
|
| 59 |
+
out._spatial_cache = input._spatial_cache
|
| 60 |
+
|
| 61 |
+
out.register_spatial_cache(f"upsample_{factor}_coords", input.coords)
|
| 62 |
+
out.register_spatial_cache(f"upsample_{factor}_layout", input.layout)
|
| 63 |
+
out.register_spatial_cache(f"upsample_{factor}_idx", idx)
|
| 64 |
+
|
| 65 |
+
return out
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
class SparseUpsample(nn.Module):
|
| 69 |
+
"""
|
| 70 |
+
Upsample a sparse tensor by a factor of `factor`.
|
| 71 |
+
Implemented as nearest neighbor interpolation.
|
| 72 |
+
"""
|
| 73 |
+
|
| 74 |
+
def __init__(self, factor: Union[int, Tuple[int, int, int], List[int]]):
|
| 75 |
+
super(SparseUpsample, self).__init__()
|
| 76 |
+
self.factor = tuple(factor) if isinstance(factor, (list, tuple)) else factor
|
| 77 |
+
|
| 78 |
+
def forward(self, input: SparseTensor) -> SparseTensor:
|
| 79 |
+
DIM = input.coords.shape[-1] - 1
|
| 80 |
+
factor = self.factor if isinstance(self.factor, tuple) else (self.factor,) * DIM
|
| 81 |
+
assert DIM == len(
|
| 82 |
+
factor
|
| 83 |
+
), "Input coordinates must have the same dimension as the upsample factor."
|
| 84 |
+
|
| 85 |
+
new_coords = input.get_spatial_cache(f"upsample_{factor}_coords")
|
| 86 |
+
new_layout = input.get_spatial_cache(f"upsample_{factor}_layout")
|
| 87 |
+
idx = input.get_spatial_cache(f"upsample_{factor}_idx")
|
| 88 |
+
if any([x is None for x in [new_coords, new_layout, idx]]):
|
| 89 |
+
raise ValueError(
|
| 90 |
+
"Upsample cache not found. SparseUpsample must be paired with SparseDownsample."
|
| 91 |
+
)
|
| 92 |
+
new_feats = input.feats[idx]
|
| 93 |
+
out = SparseTensor(new_feats, new_coords, input.shape, new_layout)
|
| 94 |
+
out._scale = tuple([s * f for s, f in zip(input._scale, factor)])
|
| 95 |
+
out._spatial_cache = input._spatial_cache
|
| 96 |
+
return out
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
class SparseSubdivide(nn.Module):
|
| 100 |
+
"""
|
| 101 |
+
Upsample a sparse tensor by a factor of `factor`.
|
| 102 |
+
Implemented as nearest neighbor interpolation.
|
| 103 |
+
"""
|
| 104 |
+
|
| 105 |
+
def __init__(self):
|
| 106 |
+
super(SparseSubdivide, self).__init__()
|
| 107 |
+
|
| 108 |
+
def forward(self, input: SparseTensor) -> SparseTensor:
|
| 109 |
+
DIM = input.coords.shape[-1] - 1
|
| 110 |
+
# upsample scale=2^DIM
|
| 111 |
+
n_cube = torch.ones([2] * DIM, device=input.device, dtype=torch.int)
|
| 112 |
+
n_coords = torch.nonzero(n_cube)
|
| 113 |
+
n_coords = torch.cat([torch.zeros_like(n_coords[:, :1]), n_coords], dim=-1)
|
| 114 |
+
factor = n_coords.shape[0]
|
| 115 |
+
assert factor == 2**DIM
|
| 116 |
+
# print(n_coords.shape)
|
| 117 |
+
new_coords = input.coords.clone()
|
| 118 |
+
new_coords[:, 1:] *= 2
|
| 119 |
+
new_coords = new_coords.unsqueeze(1) + n_coords.unsqueeze(0).to(
|
| 120 |
+
new_coords.dtype
|
| 121 |
+
)
|
| 122 |
+
|
| 123 |
+
new_feats = input.feats.unsqueeze(1).expand(
|
| 124 |
+
input.feats.shape[0], factor, *input.feats.shape[1:]
|
| 125 |
+
)
|
| 126 |
+
out = SparseTensor(
|
| 127 |
+
new_feats.flatten(0, 1), new_coords.flatten(0, 1), input.shape
|
| 128 |
+
)
|
| 129 |
+
out._scale = input._scale * 2
|
| 130 |
+
out._spatial_cache = input._spatial_cache
|
| 131 |
+
return out
|
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/transformer/__init__.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
from .blocks import *
|
| 3 |
+
from .modulated import *
|
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/transformer/blocks.py
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
from typing import *
|
| 3 |
+
import torch
|
| 4 |
+
import torch.nn as nn
|
| 5 |
+
from ..basic import SparseTensor
|
| 6 |
+
from ..linear import SparseLinear
|
| 7 |
+
from ..nonlinearity import SparseGELU
|
| 8 |
+
from ..attention import SparseMultiHeadAttention, SerializeMode
|
| 9 |
+
from ...norm import LayerNorm32
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class SparseFeedForwardNet(nn.Module):
|
| 13 |
+
def __init__(self, channels: int, mlp_ratio: float = 4.0):
|
| 14 |
+
super().__init__()
|
| 15 |
+
self.mlp = nn.Sequential(
|
| 16 |
+
SparseLinear(channels, int(channels * mlp_ratio)),
|
| 17 |
+
SparseGELU(approximate="tanh"),
|
| 18 |
+
SparseLinear(int(channels * mlp_ratio), channels),
|
| 19 |
+
)
|
| 20 |
+
|
| 21 |
+
def forward(self, x: SparseTensor) -> SparseTensor:
|
| 22 |
+
return self.mlp(x)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class SparseTransformerBlock(nn.Module):
|
| 26 |
+
"""
|
| 27 |
+
Sparse Transformer block (MSA + FFN).
|
| 28 |
+
"""
|
| 29 |
+
|
| 30 |
+
def __init__(
|
| 31 |
+
self,
|
| 32 |
+
channels: int,
|
| 33 |
+
num_heads: int,
|
| 34 |
+
mlp_ratio: float = 4.0,
|
| 35 |
+
attn_mode: Literal[
|
| 36 |
+
"full", "shift_window", "shift_sequence", "shift_order", "swin"
|
| 37 |
+
] = "full",
|
| 38 |
+
window_size: Optional[int] = None,
|
| 39 |
+
shift_sequence: Optional[int] = None,
|
| 40 |
+
shift_window: Optional[Tuple[int, int, int]] = None,
|
| 41 |
+
serialize_mode: Optional[SerializeMode] = None,
|
| 42 |
+
use_checkpoint: bool = False,
|
| 43 |
+
use_rope: bool = False,
|
| 44 |
+
qk_rms_norm: bool = False,
|
| 45 |
+
qkv_bias: bool = True,
|
| 46 |
+
ln_affine: bool = False,
|
| 47 |
+
):
|
| 48 |
+
super().__init__()
|
| 49 |
+
self.use_checkpoint = use_checkpoint
|
| 50 |
+
self.norm1 = LayerNorm32(channels, elementwise_affine=ln_affine, eps=1e-6)
|
| 51 |
+
self.norm2 = LayerNorm32(channels, elementwise_affine=ln_affine, eps=1e-6)
|
| 52 |
+
self.attn = SparseMultiHeadAttention(
|
| 53 |
+
channels,
|
| 54 |
+
num_heads=num_heads,
|
| 55 |
+
attn_mode=attn_mode,
|
| 56 |
+
window_size=window_size,
|
| 57 |
+
shift_sequence=shift_sequence,
|
| 58 |
+
shift_window=shift_window,
|
| 59 |
+
serialize_mode=serialize_mode,
|
| 60 |
+
qkv_bias=qkv_bias,
|
| 61 |
+
use_rope=use_rope,
|
| 62 |
+
qk_rms_norm=qk_rms_norm,
|
| 63 |
+
)
|
| 64 |
+
self.mlp = SparseFeedForwardNet(
|
| 65 |
+
channels,
|
| 66 |
+
mlp_ratio=mlp_ratio,
|
| 67 |
+
)
|
| 68 |
+
|
| 69 |
+
def _forward(self, x: SparseTensor) -> SparseTensor:
|
| 70 |
+
h = x.replace(self.norm1(x.feats))
|
| 71 |
+
h = self.attn(h)
|
| 72 |
+
x = x + h
|
| 73 |
+
h = x.replace(self.norm2(x.feats))
|
| 74 |
+
h = self.mlp(h)
|
| 75 |
+
x = x + h
|
| 76 |
+
return x
|
| 77 |
+
|
| 78 |
+
def forward(self, x: SparseTensor) -> SparseTensor:
|
| 79 |
+
if self.use_checkpoint:
|
| 80 |
+
return torch.utils.checkpoint.checkpoint(
|
| 81 |
+
self._forward, x, use_reentrant=False
|
| 82 |
+
)
|
| 83 |
+
else:
|
| 84 |
+
return self._forward(x)
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
class SparseTransformerCrossBlock(nn.Module):
|
| 88 |
+
"""
|
| 89 |
+
Sparse Transformer cross-attention block (MSA + MCA + FFN).
|
| 90 |
+
"""
|
| 91 |
+
|
| 92 |
+
def __init__(
|
| 93 |
+
self,
|
| 94 |
+
channels: int,
|
| 95 |
+
ctx_channels: int,
|
| 96 |
+
num_heads: int,
|
| 97 |
+
mlp_ratio: float = 4.0,
|
| 98 |
+
attn_mode: Literal[
|
| 99 |
+
"full", "shift_window", "shift_sequence", "shift_order", "swin"
|
| 100 |
+
] = "full",
|
| 101 |
+
window_size: Optional[int] = None,
|
| 102 |
+
shift_sequence: Optional[int] = None,
|
| 103 |
+
shift_window: Optional[Tuple[int, int, int]] = None,
|
| 104 |
+
serialize_mode: Optional[SerializeMode] = None,
|
| 105 |
+
use_checkpoint: bool = False,
|
| 106 |
+
use_rope: bool = False,
|
| 107 |
+
qk_rms_norm: bool = False,
|
| 108 |
+
qk_rms_norm_cross: bool = False,
|
| 109 |
+
qkv_bias: bool = True,
|
| 110 |
+
ln_affine: bool = False,
|
| 111 |
+
):
|
| 112 |
+
super().__init__()
|
| 113 |
+
self.use_checkpoint = use_checkpoint
|
| 114 |
+
self.norm1 = LayerNorm32(channels, elementwise_affine=ln_affine, eps=1e-6)
|
| 115 |
+
self.norm2 = LayerNorm32(channels, elementwise_affine=ln_affine, eps=1e-6)
|
| 116 |
+
self.norm3 = LayerNorm32(channels, elementwise_affine=ln_affine, eps=1e-6)
|
| 117 |
+
self.self_attn = SparseMultiHeadAttention(
|
| 118 |
+
channels,
|
| 119 |
+
num_heads=num_heads,
|
| 120 |
+
type="self",
|
| 121 |
+
attn_mode=attn_mode,
|
| 122 |
+
window_size=window_size,
|
| 123 |
+
shift_sequence=shift_sequence,
|
| 124 |
+
shift_window=shift_window,
|
| 125 |
+
serialize_mode=serialize_mode,
|
| 126 |
+
qkv_bias=qkv_bias,
|
| 127 |
+
use_rope=use_rope,
|
| 128 |
+
qk_rms_norm=qk_rms_norm,
|
| 129 |
+
)
|
| 130 |
+
self.cross_attn = SparseMultiHeadAttention(
|
| 131 |
+
channels,
|
| 132 |
+
ctx_channels=ctx_channels,
|
| 133 |
+
num_heads=num_heads,
|
| 134 |
+
type="cross",
|
| 135 |
+
attn_mode="full",
|
| 136 |
+
qkv_bias=qkv_bias,
|
| 137 |
+
qk_rms_norm=qk_rms_norm_cross,
|
| 138 |
+
)
|
| 139 |
+
self.mlp = SparseFeedForwardNet(
|
| 140 |
+
channels,
|
| 141 |
+
mlp_ratio=mlp_ratio,
|
| 142 |
+
)
|
| 143 |
+
|
| 144 |
+
def _forward(self, x: SparseTensor, mod: torch.Tensor, context: torch.Tensor):
|
| 145 |
+
h = x.replace(self.norm1(x.feats))
|
| 146 |
+
h = self.self_attn(h)
|
| 147 |
+
x = x + h
|
| 148 |
+
h = x.replace(self.norm2(x.feats))
|
| 149 |
+
h = self.cross_attn(h, context)
|
| 150 |
+
x = x + h
|
| 151 |
+
h = x.replace(self.norm3(x.feats))
|
| 152 |
+
h = self.mlp(h)
|
| 153 |
+
x = x + h
|
| 154 |
+
return x
|
| 155 |
+
|
| 156 |
+
def forward(self, x: SparseTensor, context: torch.Tensor):
|
| 157 |
+
if self.use_checkpoint:
|
| 158 |
+
return torch.utils.checkpoint.checkpoint(
|
| 159 |
+
self._forward, x, context, use_reentrant=False
|
| 160 |
+
)
|
| 161 |
+
else:
|
| 162 |
+
return self._forward(x, context)
|
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/sparse/transformer/modulated.py
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
from typing import *
|
| 3 |
+
import torch
|
| 4 |
+
import torch.nn as nn
|
| 5 |
+
from ..basic import SparseTensor
|
| 6 |
+
from ..attention import SparseMultiHeadAttention, SerializeMode
|
| 7 |
+
from ...norm import LayerNorm32
|
| 8 |
+
from .blocks import SparseFeedForwardNet
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class ModulatedSparseTransformerBlock(nn.Module):
|
| 12 |
+
"""
|
| 13 |
+
Sparse Transformer block (MSA + FFN) with adaptive layer norm conditioning.
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
def __init__(
|
| 17 |
+
self,
|
| 18 |
+
channels: int,
|
| 19 |
+
num_heads: int,
|
| 20 |
+
mlp_ratio: float = 4.0,
|
| 21 |
+
attn_mode: Literal[
|
| 22 |
+
"full", "shift_window", "shift_sequence", "shift_order", "swin"
|
| 23 |
+
] = "full",
|
| 24 |
+
window_size: Optional[int] = None,
|
| 25 |
+
shift_sequence: Optional[int] = None,
|
| 26 |
+
shift_window: Optional[Tuple[int, int, int]] = None,
|
| 27 |
+
serialize_mode: Optional[SerializeMode] = None,
|
| 28 |
+
use_checkpoint: bool = False,
|
| 29 |
+
use_rope: bool = False,
|
| 30 |
+
qk_rms_norm: bool = False,
|
| 31 |
+
qkv_bias: bool = True,
|
| 32 |
+
share_mod: bool = False,
|
| 33 |
+
):
|
| 34 |
+
super().__init__()
|
| 35 |
+
self.use_checkpoint = use_checkpoint
|
| 36 |
+
self.share_mod = share_mod
|
| 37 |
+
self.norm1 = LayerNorm32(channels, elementwise_affine=False, eps=1e-6)
|
| 38 |
+
self.norm2 = LayerNorm32(channels, elementwise_affine=False, eps=1e-6)
|
| 39 |
+
self.attn = SparseMultiHeadAttention(
|
| 40 |
+
channels,
|
| 41 |
+
num_heads=num_heads,
|
| 42 |
+
attn_mode=attn_mode,
|
| 43 |
+
window_size=window_size,
|
| 44 |
+
shift_sequence=shift_sequence,
|
| 45 |
+
shift_window=shift_window,
|
| 46 |
+
serialize_mode=serialize_mode,
|
| 47 |
+
qkv_bias=qkv_bias,
|
| 48 |
+
use_rope=use_rope,
|
| 49 |
+
qk_rms_norm=qk_rms_norm,
|
| 50 |
+
)
|
| 51 |
+
self.mlp = SparseFeedForwardNet(
|
| 52 |
+
channels,
|
| 53 |
+
mlp_ratio=mlp_ratio,
|
| 54 |
+
)
|
| 55 |
+
if not share_mod:
|
| 56 |
+
self.adaLN_modulation = nn.Sequential(
|
| 57 |
+
nn.SiLU(), nn.Linear(channels, 6 * channels, bias=True)
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
def _forward(self, x: SparseTensor, mod: torch.Tensor) -> SparseTensor:
|
| 61 |
+
if self.share_mod:
|
| 62 |
+
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mod.chunk(
|
| 63 |
+
6, dim=1
|
| 64 |
+
)
|
| 65 |
+
else:
|
| 66 |
+
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
|
| 67 |
+
self.adaLN_modulation(mod).chunk(6, dim=1)
|
| 68 |
+
)
|
| 69 |
+
h = x.replace(self.norm1(x.feats))
|
| 70 |
+
h = h * (1 + scale_msa) + shift_msa
|
| 71 |
+
h = self.attn(h)
|
| 72 |
+
h = h * gate_msa
|
| 73 |
+
x = x + h
|
| 74 |
+
h = x.replace(self.norm2(x.feats))
|
| 75 |
+
h = h * (1 + scale_mlp) + shift_mlp
|
| 76 |
+
h = self.mlp(h)
|
| 77 |
+
h = h * gate_mlp
|
| 78 |
+
x = x + h
|
| 79 |
+
return x
|
| 80 |
+
|
| 81 |
+
def forward(self, x: SparseTensor, mod: torch.Tensor) -> SparseTensor:
|
| 82 |
+
if self.use_checkpoint:
|
| 83 |
+
return torch.utils.checkpoint.checkpoint(
|
| 84 |
+
self._forward, x, mod, use_reentrant=False
|
| 85 |
+
)
|
| 86 |
+
else:
|
| 87 |
+
return self._forward(x, mod)
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
class ModulatedSparseTransformerCrossBlock(nn.Module):
|
| 91 |
+
"""
|
| 92 |
+
Sparse Transformer cross-attention block (MSA + MCA + FFN) with adaptive layer norm conditioning.
|
| 93 |
+
"""
|
| 94 |
+
|
| 95 |
+
def __init__(
|
| 96 |
+
self,
|
| 97 |
+
channels: int,
|
| 98 |
+
ctx_channels: int,
|
| 99 |
+
num_heads: int,
|
| 100 |
+
mlp_ratio: float = 4.0,
|
| 101 |
+
attn_mode: Literal[
|
| 102 |
+
"full", "shift_window", "shift_sequence", "shift_order", "swin"
|
| 103 |
+
] = "full",
|
| 104 |
+
window_size: Optional[int] = None,
|
| 105 |
+
shift_sequence: Optional[int] = None,
|
| 106 |
+
shift_window: Optional[Tuple[int, int, int]] = None,
|
| 107 |
+
serialize_mode: Optional[SerializeMode] = None,
|
| 108 |
+
use_checkpoint: bool = False,
|
| 109 |
+
use_rope: bool = False,
|
| 110 |
+
qk_rms_norm: bool = False,
|
| 111 |
+
qk_rms_norm_cross: bool = False,
|
| 112 |
+
qkv_bias: bool = True,
|
| 113 |
+
share_mod: bool = False,
|
| 114 |
+
):
|
| 115 |
+
super().__init__()
|
| 116 |
+
self.use_checkpoint = use_checkpoint
|
| 117 |
+
self.share_mod = share_mod
|
| 118 |
+
self.norm1 = LayerNorm32(channels, elementwise_affine=False, eps=1e-6)
|
| 119 |
+
self.norm2 = LayerNorm32(channels, elementwise_affine=True, eps=1e-6)
|
| 120 |
+
self.norm3 = LayerNorm32(channels, elementwise_affine=False, eps=1e-6)
|
| 121 |
+
self.self_attn = SparseMultiHeadAttention(
|
| 122 |
+
channels,
|
| 123 |
+
num_heads=num_heads,
|
| 124 |
+
type="self",
|
| 125 |
+
attn_mode=attn_mode,
|
| 126 |
+
window_size=window_size,
|
| 127 |
+
shift_sequence=shift_sequence,
|
| 128 |
+
shift_window=shift_window,
|
| 129 |
+
serialize_mode=serialize_mode,
|
| 130 |
+
qkv_bias=qkv_bias,
|
| 131 |
+
use_rope=use_rope,
|
| 132 |
+
qk_rms_norm=qk_rms_norm,
|
| 133 |
+
)
|
| 134 |
+
self.cross_attn = SparseMultiHeadAttention(
|
| 135 |
+
channels,
|
| 136 |
+
ctx_channels=ctx_channels,
|
| 137 |
+
num_heads=num_heads,
|
| 138 |
+
type="cross",
|
| 139 |
+
attn_mode="full",
|
| 140 |
+
qkv_bias=qkv_bias,
|
| 141 |
+
qk_rms_norm=qk_rms_norm_cross,
|
| 142 |
+
)
|
| 143 |
+
self.mlp = SparseFeedForwardNet(
|
| 144 |
+
channels,
|
| 145 |
+
mlp_ratio=mlp_ratio,
|
| 146 |
+
)
|
| 147 |
+
if not share_mod:
|
| 148 |
+
self.adaLN_modulation = nn.Sequential(
|
| 149 |
+
nn.SiLU(), nn.Linear(channels, 6 * channels, bias=True)
|
| 150 |
+
)
|
| 151 |
+
|
| 152 |
+
def _forward(
|
| 153 |
+
self, x: SparseTensor, mod: torch.Tensor, context: torch.Tensor
|
| 154 |
+
) -> SparseTensor:
|
| 155 |
+
if self.share_mod:
|
| 156 |
+
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mod.chunk(
|
| 157 |
+
6, dim=1
|
| 158 |
+
)
|
| 159 |
+
else:
|
| 160 |
+
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
|
| 161 |
+
self.adaLN_modulation(mod).chunk(6, dim=1)
|
| 162 |
+
)
|
| 163 |
+
h = x.replace(self.norm1(x.feats))
|
| 164 |
+
h = h * (1 + scale_msa) + shift_msa
|
| 165 |
+
h = self.self_attn(h)
|
| 166 |
+
h = h * gate_msa
|
| 167 |
+
x = x + h
|
| 168 |
+
h = x.replace(self.norm2(x.feats))
|
| 169 |
+
h = self.cross_attn(h, context)
|
| 170 |
+
x = x + h
|
| 171 |
+
h = x.replace(self.norm3(x.feats))
|
| 172 |
+
h = h * (1 + scale_mlp) + shift_mlp
|
| 173 |
+
h = self.mlp(h)
|
| 174 |
+
h = h * gate_mlp
|
| 175 |
+
x = x + h
|
| 176 |
+
return x
|
| 177 |
+
|
| 178 |
+
def forward(
|
| 179 |
+
self, x: SparseTensor, mod: torch.Tensor, context: torch.Tensor
|
| 180 |
+
) -> SparseTensor:
|
| 181 |
+
if self.use_checkpoint:
|
| 182 |
+
return torch.utils.checkpoint.checkpoint(
|
| 183 |
+
self._forward, x, mod, context, use_reentrant=False
|
| 184 |
+
)
|
| 185 |
+
else:
|
| 186 |
+
return self._forward(x, mod, context)
|
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/spatial.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
import torch
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
def pixel_shuffle_3d(x: torch.Tensor, scale_factor: int) -> torch.Tensor:
|
| 6 |
+
"""
|
| 7 |
+
3D pixel shuffle.
|
| 8 |
+
"""
|
| 9 |
+
B, C, H, W, D = x.shape
|
| 10 |
+
C_ = C // scale_factor**3
|
| 11 |
+
x = x.reshape(B, C_, scale_factor, scale_factor, scale_factor, H, W, D)
|
| 12 |
+
x = x.permute(0, 1, 5, 2, 6, 3, 7, 4)
|
| 13 |
+
x = x.reshape(B, C_, H * scale_factor, W * scale_factor, D * scale_factor)
|
| 14 |
+
return x
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def patchify(x: torch.Tensor, patch_size: int):
|
| 18 |
+
"""
|
| 19 |
+
Patchify a tensor.
|
| 20 |
+
|
| 21 |
+
Args:
|
| 22 |
+
x (torch.Tensor): (N, C, *spatial) tensor
|
| 23 |
+
patch_size (int): Patch size
|
| 24 |
+
"""
|
| 25 |
+
DIM = x.dim() - 2
|
| 26 |
+
for d in range(2, DIM + 2):
|
| 27 |
+
assert (
|
| 28 |
+
x.shape[d] % patch_size == 0
|
| 29 |
+
), f"Dimension {d} of input tensor must be divisible by patch size, got {x.shape[d]} and {patch_size}"
|
| 30 |
+
|
| 31 |
+
x = x.reshape(
|
| 32 |
+
*x.shape[:2],
|
| 33 |
+
*sum([[x.shape[d] // patch_size, patch_size] for d in range(2, DIM + 2)], []),
|
| 34 |
+
)
|
| 35 |
+
x = x.permute(
|
| 36 |
+
0, 1, *([2 * i + 3 for i in range(DIM)] + [2 * i + 2 for i in range(DIM)])
|
| 37 |
+
)
|
| 38 |
+
x = x.reshape(x.shape[0], x.shape[1] * (patch_size**DIM), *(x.shape[-DIM:]))
|
| 39 |
+
return x
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def unpatchify(x: torch.Tensor, patch_size: int):
|
| 43 |
+
"""
|
| 44 |
+
Unpatchify a tensor.
|
| 45 |
+
|
| 46 |
+
Args:
|
| 47 |
+
x (torch.Tensor): (N, C, *spatial) tensor
|
| 48 |
+
patch_size (int): Patch size
|
| 49 |
+
"""
|
| 50 |
+
DIM = x.dim() - 2
|
| 51 |
+
assert (
|
| 52 |
+
x.shape[1] % (patch_size**DIM) == 0
|
| 53 |
+
), f"Second dimension of input tensor must be divisible by patch size to unpatchify, got {x.shape[1]} and {patch_size ** DIM}"
|
| 54 |
+
|
| 55 |
+
x = x.reshape(
|
| 56 |
+
x.shape[0],
|
| 57 |
+
x.shape[1] // (patch_size**DIM),
|
| 58 |
+
*([patch_size] * DIM),
|
| 59 |
+
*(x.shape[-DIM:]),
|
| 60 |
+
)
|
| 61 |
+
x = x.permute(0, 1, *(sum([[2 + DIM + i, 2 + i] for i in range(DIM)], [])))
|
| 62 |
+
x = x.reshape(
|
| 63 |
+
x.shape[0], x.shape[1], *[x.shape[2 + 2 * i] * patch_size for i in range(DIM)]
|
| 64 |
+
)
|
| 65 |
+
return x
|
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/transformer/__init__.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
from .blocks import *
|
| 3 |
+
from .modulated import *
|
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/transformer/blocks.py
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
from typing import *
|
| 3 |
+
import torch
|
| 4 |
+
import torch.nn as nn
|
| 5 |
+
from ..attention import MultiHeadAttention
|
| 6 |
+
from ..norm import LayerNorm32
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class AbsolutePositionEmbedder(nn.Module):
|
| 10 |
+
"""
|
| 11 |
+
Embeds spatial positions into vector representations.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
def __init__(self, channels: int, in_channels: int = 3):
|
| 15 |
+
super().__init__()
|
| 16 |
+
self.channels = channels
|
| 17 |
+
self.in_channels = in_channels
|
| 18 |
+
self.freq_dim = channels // in_channels // 2
|
| 19 |
+
self.freqs = torch.arange(self.freq_dim, dtype=torch.float32) / self.freq_dim
|
| 20 |
+
self.freqs = 1.0 / (10000**self.freqs)
|
| 21 |
+
|
| 22 |
+
def _sin_cos_embedding(self, x: torch.Tensor) -> torch.Tensor:
|
| 23 |
+
"""
|
| 24 |
+
Create sinusoidal position embeddings.
|
| 25 |
+
|
| 26 |
+
Args:
|
| 27 |
+
x: a 1-D Tensor of N indices
|
| 28 |
+
|
| 29 |
+
Returns:
|
| 30 |
+
an (N, D) Tensor of positional embeddings.
|
| 31 |
+
"""
|
| 32 |
+
self.freqs = self.freqs.to(x.device)
|
| 33 |
+
out = torch.outer(x, self.freqs)
|
| 34 |
+
out = torch.cat([torch.sin(out), torch.cos(out)], dim=-1)
|
| 35 |
+
return out
|
| 36 |
+
|
| 37 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 38 |
+
"""
|
| 39 |
+
Args:
|
| 40 |
+
x (torch.Tensor): (N, D) tensor of spatial positions
|
| 41 |
+
"""
|
| 42 |
+
N, D = x.shape
|
| 43 |
+
assert (
|
| 44 |
+
D == self.in_channels
|
| 45 |
+
), "Input dimension must match number of input channels"
|
| 46 |
+
embed = self._sin_cos_embedding(x.reshape(-1))
|
| 47 |
+
embed = embed.reshape(N, -1)
|
| 48 |
+
if embed.shape[1] < self.channels:
|
| 49 |
+
embed = torch.cat(
|
| 50 |
+
[
|
| 51 |
+
embed,
|
| 52 |
+
torch.zeros(N, self.channels - embed.shape[1], device=embed.device),
|
| 53 |
+
],
|
| 54 |
+
dim=-1,
|
| 55 |
+
)
|
| 56 |
+
return embed
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
class FeedForwardNet(nn.Module):
|
| 60 |
+
def __init__(self, channels: int, mlp_ratio: float = 4.0):
|
| 61 |
+
super().__init__()
|
| 62 |
+
self.mlp = nn.Sequential(
|
| 63 |
+
nn.Linear(channels, int(channels * mlp_ratio)),
|
| 64 |
+
nn.GELU(approximate="tanh"),
|
| 65 |
+
nn.Linear(int(channels * mlp_ratio), channels),
|
| 66 |
+
)
|
| 67 |
+
|
| 68 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 69 |
+
return self.mlp(x)
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
class TransformerBlock(nn.Module):
|
| 73 |
+
"""
|
| 74 |
+
Transformer block (MSA + FFN).
|
| 75 |
+
"""
|
| 76 |
+
|
| 77 |
+
def __init__(
|
| 78 |
+
self,
|
| 79 |
+
channels: int,
|
| 80 |
+
num_heads: int,
|
| 81 |
+
mlp_ratio: float = 4.0,
|
| 82 |
+
attn_mode: Literal["full", "windowed"] = "full",
|
| 83 |
+
window_size: Optional[int] = None,
|
| 84 |
+
shift_window: Optional[int] = None,
|
| 85 |
+
use_checkpoint: bool = False,
|
| 86 |
+
use_rope: bool = False,
|
| 87 |
+
qk_rms_norm: bool = False,
|
| 88 |
+
qkv_bias: bool = True,
|
| 89 |
+
ln_affine: bool = False,
|
| 90 |
+
):
|
| 91 |
+
super().__init__()
|
| 92 |
+
self.use_checkpoint = use_checkpoint
|
| 93 |
+
self.norm1 = LayerNorm32(channels, elementwise_affine=ln_affine, eps=1e-6)
|
| 94 |
+
self.norm2 = LayerNorm32(channels, elementwise_affine=ln_affine, eps=1e-6)
|
| 95 |
+
self.attn = MultiHeadAttention(
|
| 96 |
+
channels,
|
| 97 |
+
num_heads=num_heads,
|
| 98 |
+
attn_mode=attn_mode,
|
| 99 |
+
window_size=window_size,
|
| 100 |
+
shift_window=shift_window,
|
| 101 |
+
qkv_bias=qkv_bias,
|
| 102 |
+
use_rope=use_rope,
|
| 103 |
+
qk_rms_norm=qk_rms_norm,
|
| 104 |
+
)
|
| 105 |
+
self.mlp = FeedForwardNet(
|
| 106 |
+
channels,
|
| 107 |
+
mlp_ratio=mlp_ratio,
|
| 108 |
+
)
|
| 109 |
+
|
| 110 |
+
def _forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 111 |
+
h = self.norm1(x)
|
| 112 |
+
h = self.attn(h)
|
| 113 |
+
x = x + h
|
| 114 |
+
h = self.norm2(x)
|
| 115 |
+
h = self.mlp(h)
|
| 116 |
+
x = x + h
|
| 117 |
+
return x
|
| 118 |
+
|
| 119 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 120 |
+
if self.use_checkpoint:
|
| 121 |
+
return torch.utils.checkpoint.checkpoint(
|
| 122 |
+
self._forward, x, use_reentrant=False
|
| 123 |
+
)
|
| 124 |
+
else:
|
| 125 |
+
return self._forward(x)
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
class TransformerCrossBlock(nn.Module):
|
| 129 |
+
"""
|
| 130 |
+
Transformer cross-attention block (MSA + MCA + FFN).
|
| 131 |
+
"""
|
| 132 |
+
|
| 133 |
+
def __init__(
|
| 134 |
+
self,
|
| 135 |
+
channels: int,
|
| 136 |
+
ctx_channels: int,
|
| 137 |
+
num_heads: int,
|
| 138 |
+
mlp_ratio: float = 4.0,
|
| 139 |
+
attn_mode: Literal["full", "windowed"] = "full",
|
| 140 |
+
window_size: Optional[int] = None,
|
| 141 |
+
shift_window: Optional[Tuple[int, int, int]] = None,
|
| 142 |
+
use_checkpoint: bool = False,
|
| 143 |
+
use_rope: bool = False,
|
| 144 |
+
qk_rms_norm: bool = False,
|
| 145 |
+
qk_rms_norm_cross: bool = False,
|
| 146 |
+
qkv_bias: bool = True,
|
| 147 |
+
ln_affine: bool = False,
|
| 148 |
+
):
|
| 149 |
+
super().__init__()
|
| 150 |
+
self.use_checkpoint = use_checkpoint
|
| 151 |
+
self.norm1 = LayerNorm32(channels, elementwise_affine=ln_affine, eps=1e-6)
|
| 152 |
+
self.norm2 = LayerNorm32(channels, elementwise_affine=ln_affine, eps=1e-6)
|
| 153 |
+
self.norm3 = LayerNorm32(channels, elementwise_affine=ln_affine, eps=1e-6)
|
| 154 |
+
self.self_attn = MultiHeadAttention(
|
| 155 |
+
channels,
|
| 156 |
+
num_heads=num_heads,
|
| 157 |
+
type="self",
|
| 158 |
+
attn_mode=attn_mode,
|
| 159 |
+
window_size=window_size,
|
| 160 |
+
shift_window=shift_window,
|
| 161 |
+
qkv_bias=qkv_bias,
|
| 162 |
+
use_rope=use_rope,
|
| 163 |
+
qk_rms_norm=qk_rms_norm,
|
| 164 |
+
)
|
| 165 |
+
self.cross_attn = MultiHeadAttention(
|
| 166 |
+
channels,
|
| 167 |
+
ctx_channels=ctx_channels,
|
| 168 |
+
num_heads=num_heads,
|
| 169 |
+
type="cross",
|
| 170 |
+
attn_mode="full",
|
| 171 |
+
qkv_bias=qkv_bias,
|
| 172 |
+
qk_rms_norm=qk_rms_norm_cross,
|
| 173 |
+
)
|
| 174 |
+
self.mlp = FeedForwardNet(
|
| 175 |
+
channels,
|
| 176 |
+
mlp_ratio=mlp_ratio,
|
| 177 |
+
)
|
| 178 |
+
|
| 179 |
+
def _forward(self, x: torch.Tensor, context: torch.Tensor):
|
| 180 |
+
h = self.norm1(x)
|
| 181 |
+
h = self.self_attn(h)
|
| 182 |
+
x = x + h
|
| 183 |
+
h = self.norm2(x)
|
| 184 |
+
h = self.cross_attn(h, context)
|
| 185 |
+
x = x + h
|
| 186 |
+
h = self.norm3(x)
|
| 187 |
+
h = self.mlp(h)
|
| 188 |
+
x = x + h
|
| 189 |
+
return x
|
| 190 |
+
|
| 191 |
+
def forward(self, x: torch.Tensor, context: torch.Tensor):
|
| 192 |
+
if self.use_checkpoint:
|
| 193 |
+
return torch.utils.checkpoint.checkpoint(
|
| 194 |
+
self._forward, x, context, use_reentrant=False
|
| 195 |
+
)
|
| 196 |
+
else:
|
| 197 |
+
return self._forward(x, context)
|
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/transformer/modulated.py
ADDED
|
@@ -0,0 +1,341 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
from functools import partial
|
| 3 |
+
from typing import *
|
| 4 |
+
from torch.utils import _pytree
|
| 5 |
+
import torch
|
| 6 |
+
import torch.nn as nn
|
| 7 |
+
from ..attention import MultiHeadAttention, MOTMultiHeadSelfAttention
|
| 8 |
+
from ..norm import LayerNorm32
|
| 9 |
+
from .blocks import FeedForwardNet
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class ModulatedTransformerBlock(nn.Module):
|
| 13 |
+
"""
|
| 14 |
+
Transformer block (MSA + FFN) with adaptive layer norm conditioning.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
def __init__(
|
| 18 |
+
self,
|
| 19 |
+
channels: int,
|
| 20 |
+
num_heads: int,
|
| 21 |
+
mlp_ratio: float = 4.0,
|
| 22 |
+
attn_mode: Literal["full", "windowed"] = "full",
|
| 23 |
+
window_size: Optional[int] = None,
|
| 24 |
+
shift_window: Optional[Tuple[int, int, int]] = None,
|
| 25 |
+
use_checkpoint: bool = False,
|
| 26 |
+
use_rope: bool = False,
|
| 27 |
+
qk_rms_norm: bool = False,
|
| 28 |
+
qkv_bias: bool = True,
|
| 29 |
+
share_mod: bool = False,
|
| 30 |
+
):
|
| 31 |
+
super().__init__()
|
| 32 |
+
self.use_checkpoint = use_checkpoint
|
| 33 |
+
self.share_mod = share_mod
|
| 34 |
+
self.norm1 = LayerNorm32(channels, elementwise_affine=False, eps=1e-6)
|
| 35 |
+
self.norm2 = LayerNorm32(channels, elementwise_affine=False, eps=1e-6)
|
| 36 |
+
self.attn = MultiHeadAttention(
|
| 37 |
+
channels,
|
| 38 |
+
num_heads=num_heads,
|
| 39 |
+
attn_mode=attn_mode,
|
| 40 |
+
window_size=window_size,
|
| 41 |
+
shift_window=shift_window,
|
| 42 |
+
qkv_bias=qkv_bias,
|
| 43 |
+
use_rope=use_rope,
|
| 44 |
+
qk_rms_norm=qk_rms_norm,
|
| 45 |
+
)
|
| 46 |
+
self.mlp = FeedForwardNet(
|
| 47 |
+
channels,
|
| 48 |
+
mlp_ratio=mlp_ratio,
|
| 49 |
+
)
|
| 50 |
+
if not share_mod:
|
| 51 |
+
self.adaLN_modulation = nn.Sequential(
|
| 52 |
+
nn.SiLU(), nn.Linear(channels, 6 * channels, bias=True)
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
def _forward(self, x: torch.Tensor, mod: torch.Tensor) -> torch.Tensor:
|
| 56 |
+
if self.share_mod:
|
| 57 |
+
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mod.chunk(
|
| 58 |
+
6, dim=1
|
| 59 |
+
)
|
| 60 |
+
else:
|
| 61 |
+
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
|
| 62 |
+
self.adaLN_modulation(mod).chunk(6, dim=1)
|
| 63 |
+
)
|
| 64 |
+
h = self.norm1(x)
|
| 65 |
+
h = h * (1 + scale_msa.unsqueeze(1)) + shift_msa.unsqueeze(1)
|
| 66 |
+
h = self.attn(h)
|
| 67 |
+
h = h * gate_msa.unsqueeze(1)
|
| 68 |
+
x = x + h
|
| 69 |
+
h = self.norm2(x)
|
| 70 |
+
h = h * (1 + scale_mlp.unsqueeze(1)) + shift_mlp.unsqueeze(1)
|
| 71 |
+
h = self.mlp(h)
|
| 72 |
+
h = h * gate_mlp.unsqueeze(1)
|
| 73 |
+
x = x + h
|
| 74 |
+
return x
|
| 75 |
+
|
| 76 |
+
def forward(self, x: torch.Tensor, mod: torch.Tensor) -> torch.Tensor:
|
| 77 |
+
if self.use_checkpoint:
|
| 78 |
+
return torch.utils.checkpoint.checkpoint(
|
| 79 |
+
self._forward, x, mod, use_reentrant=False
|
| 80 |
+
)
|
| 81 |
+
else:
|
| 82 |
+
return self._forward(x, mod)
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
class ModulatedTransformerCrossBlock(nn.Module):
|
| 86 |
+
"""
|
| 87 |
+
Transformer cross-attention block (MSA + MCA + FFN) with adaptive layer norm conditioning.
|
| 88 |
+
"""
|
| 89 |
+
|
| 90 |
+
def __init__(
|
| 91 |
+
self,
|
| 92 |
+
channels: int,
|
| 93 |
+
ctx_channels: int,
|
| 94 |
+
num_heads: int,
|
| 95 |
+
mlp_ratio: float = 4.0,
|
| 96 |
+
attn_mode: Literal["full", "windowed"] = "full",
|
| 97 |
+
window_size: Optional[int] = None,
|
| 98 |
+
shift_window: Optional[Tuple[int, int, int]] = None,
|
| 99 |
+
use_checkpoint: bool = False,
|
| 100 |
+
use_rope: bool = False,
|
| 101 |
+
qk_rms_norm: bool = False,
|
| 102 |
+
qk_rms_norm_cross: bool = False,
|
| 103 |
+
qkv_bias: bool = True,
|
| 104 |
+
share_mod: bool = False,
|
| 105 |
+
):
|
| 106 |
+
super().__init__()
|
| 107 |
+
self.use_checkpoint = use_checkpoint
|
| 108 |
+
self.share_mod = share_mod
|
| 109 |
+
self.norm1 = LayerNorm32(channels, elementwise_affine=False, eps=1e-6)
|
| 110 |
+
self.norm2 = LayerNorm32(channels, elementwise_affine=True, eps=1e-6)
|
| 111 |
+
self.norm3 = LayerNorm32(channels, elementwise_affine=False, eps=1e-6)
|
| 112 |
+
self.self_attn = MultiHeadAttention(
|
| 113 |
+
channels,
|
| 114 |
+
num_heads=num_heads,
|
| 115 |
+
type="self",
|
| 116 |
+
attn_mode=attn_mode,
|
| 117 |
+
window_size=window_size,
|
| 118 |
+
shift_window=shift_window,
|
| 119 |
+
qkv_bias=qkv_bias,
|
| 120 |
+
use_rope=use_rope,
|
| 121 |
+
qk_rms_norm=qk_rms_norm,
|
| 122 |
+
)
|
| 123 |
+
self.cross_attn = MultiHeadAttention(
|
| 124 |
+
channels,
|
| 125 |
+
ctx_channels=ctx_channels,
|
| 126 |
+
num_heads=num_heads,
|
| 127 |
+
type="cross",
|
| 128 |
+
attn_mode="full",
|
| 129 |
+
qkv_bias=qkv_bias,
|
| 130 |
+
qk_rms_norm=qk_rms_norm_cross,
|
| 131 |
+
)
|
| 132 |
+
self.mlp = FeedForwardNet(
|
| 133 |
+
channels,
|
| 134 |
+
mlp_ratio=mlp_ratio,
|
| 135 |
+
)
|
| 136 |
+
if not share_mod:
|
| 137 |
+
self.adaLN_modulation = nn.Sequential(
|
| 138 |
+
nn.SiLU(), nn.Linear(channels, 6 * channels, bias=True)
|
| 139 |
+
)
|
| 140 |
+
|
| 141 |
+
def _forward(self, x: torch.Tensor, mod: torch.Tensor, context: torch.Tensor):
|
| 142 |
+
if self.share_mod:
|
| 143 |
+
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mod.chunk(
|
| 144 |
+
6, dim=1
|
| 145 |
+
)
|
| 146 |
+
else:
|
| 147 |
+
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
|
| 148 |
+
self.adaLN_modulation(mod).chunk(6, dim=1)
|
| 149 |
+
)
|
| 150 |
+
h = self.norm1(x)
|
| 151 |
+
h = h * (1 + scale_msa.unsqueeze(1)) + shift_msa.unsqueeze(1)
|
| 152 |
+
h = self.self_attn(h)
|
| 153 |
+
h = h * gate_msa.unsqueeze(1)
|
| 154 |
+
x = x + h
|
| 155 |
+
h = self.norm2(x)
|
| 156 |
+
h = self.cross_attn(h, context)
|
| 157 |
+
x = x + h
|
| 158 |
+
h = self.norm3(x)
|
| 159 |
+
h = h * (1 + scale_mlp.unsqueeze(1)) + shift_mlp.unsqueeze(1)
|
| 160 |
+
h = self.mlp(h)
|
| 161 |
+
h = h * gate_mlp.unsqueeze(1)
|
| 162 |
+
x = x + h
|
| 163 |
+
return x
|
| 164 |
+
|
| 165 |
+
def forward(self, x: torch.Tensor, mod: torch.Tensor, context: torch.Tensor):
|
| 166 |
+
if self.use_checkpoint:
|
| 167 |
+
return torch.utils.checkpoint.checkpoint(
|
| 168 |
+
self._forward, x, mod, context, use_reentrant=False
|
| 169 |
+
)
|
| 170 |
+
else:
|
| 171 |
+
return self._forward(x, mod, context)
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
class MOTModulatedTransformerCrossBlock(nn.Module):
|
| 175 |
+
"""
|
| 176 |
+
Transformer cross-attention block (MSA + MCA + FFN) with adaptive layer norm conditioning.
|
| 177 |
+
"""
|
| 178 |
+
|
| 179 |
+
def __init__(
|
| 180 |
+
self,
|
| 181 |
+
channels: int,
|
| 182 |
+
ctx_channels: int,
|
| 183 |
+
num_heads: int,
|
| 184 |
+
mlp_ratio: float = 4.0,
|
| 185 |
+
attn_mode: Literal["full", "windowed"] = "full",
|
| 186 |
+
window_size: Optional[int] = None,
|
| 187 |
+
shift_window: Optional[Tuple[int, int, int]] = None,
|
| 188 |
+
use_checkpoint: bool = False,
|
| 189 |
+
use_rope: bool = False,
|
| 190 |
+
qk_rms_norm: bool = False,
|
| 191 |
+
qk_rms_norm_cross: bool = False,
|
| 192 |
+
qkv_bias: bool = True,
|
| 193 |
+
share_mod: bool = False,
|
| 194 |
+
latent_names: List = None,
|
| 195 |
+
freeze_shared_parameters: bool = False,
|
| 196 |
+
):
|
| 197 |
+
super().__init__()
|
| 198 |
+
self.use_checkpoint = use_checkpoint
|
| 199 |
+
self.share_mod = share_mod
|
| 200 |
+
self.norm1 = torch.nn.ModuleDict(
|
| 201 |
+
{
|
| 202 |
+
latent_name: LayerNorm32(channels, elementwise_affine=False, eps=1e-6)
|
| 203 |
+
for latent_name in latent_names
|
| 204 |
+
}
|
| 205 |
+
)
|
| 206 |
+
self.norm2 = torch.nn.ModuleDict(
|
| 207 |
+
{
|
| 208 |
+
latent_name: LayerNorm32(channels, elementwise_affine=True, eps=1e-6)
|
| 209 |
+
for latent_name in latent_names
|
| 210 |
+
}
|
| 211 |
+
)
|
| 212 |
+
self.norm3 = torch.nn.ModuleDict(
|
| 213 |
+
{
|
| 214 |
+
latent_name: LayerNorm32(channels, elementwise_affine=False, eps=1e-6)
|
| 215 |
+
for latent_name in latent_names
|
| 216 |
+
}
|
| 217 |
+
)
|
| 218 |
+
self.self_attn = MOTMultiHeadSelfAttention(
|
| 219 |
+
channels,
|
| 220 |
+
num_heads=num_heads,
|
| 221 |
+
type="self",
|
| 222 |
+
attn_mode=attn_mode,
|
| 223 |
+
window_size=window_size,
|
| 224 |
+
shift_window=shift_window,
|
| 225 |
+
qkv_bias=qkv_bias,
|
| 226 |
+
use_rope=use_rope,
|
| 227 |
+
qk_rms_norm=qk_rms_norm,
|
| 228 |
+
latent_names=latent_names,
|
| 229 |
+
)
|
| 230 |
+
self.cross_attn = torch.nn.ModuleDict(
|
| 231 |
+
{
|
| 232 |
+
latent_name: MultiHeadAttention(
|
| 233 |
+
channels,
|
| 234 |
+
ctx_channels=ctx_channels,
|
| 235 |
+
num_heads=num_heads,
|
| 236 |
+
type="cross",
|
| 237 |
+
attn_mode="full",
|
| 238 |
+
qkv_bias=qkv_bias,
|
| 239 |
+
qk_rms_norm=qk_rms_norm_cross,
|
| 240 |
+
)
|
| 241 |
+
for latent_name in latent_names
|
| 242 |
+
}
|
| 243 |
+
)
|
| 244 |
+
self.mlp = torch.nn.ModuleDict(
|
| 245 |
+
{
|
| 246 |
+
latent_name: FeedForwardNet(
|
| 247 |
+
channels,
|
| 248 |
+
mlp_ratio=mlp_ratio,
|
| 249 |
+
)
|
| 250 |
+
for latent_name in latent_names
|
| 251 |
+
}
|
| 252 |
+
)
|
| 253 |
+
if not share_mod:
|
| 254 |
+
self.adaLN_modulation = nn.Sequential(
|
| 255 |
+
nn.SiLU(), nn.Linear(channels, 6 * channels, bias=True)
|
| 256 |
+
)
|
| 257 |
+
if freeze_shared_parameters:
|
| 258 |
+
self.adaLN_modulation.eval()
|
| 259 |
+
self.adaLN_modulation.requires_grad_(False)
|
| 260 |
+
|
| 261 |
+
def _apply_module(self, h, module):
|
| 262 |
+
return module(h)
|
| 263 |
+
|
| 264 |
+
def _apply_cross_attn(self, h, cross_attn, context):
|
| 265 |
+
return cross_attn(h, context)
|
| 266 |
+
|
| 267 |
+
def _apply_msa(self, h, scale_msa, shift_msa):
|
| 268 |
+
return h * (1 + scale_msa.unsqueeze(1)) + shift_msa.unsqueeze(1)
|
| 269 |
+
|
| 270 |
+
def _apply_mlp(self, h, scale_mlp, shift_mlp):
|
| 271 |
+
return h * (1 + scale_mlp.unsqueeze(1)) + shift_mlp.unsqueeze(1)
|
| 272 |
+
|
| 273 |
+
def _apply_add(self, x, h):
|
| 274 |
+
return x + h
|
| 275 |
+
|
| 276 |
+
def _apply_multiplication(self, h, multiplier):
|
| 277 |
+
return h * multiplier.unsqueeze(1)
|
| 278 |
+
|
| 279 |
+
# This is stupid, _pytree does not support ModuleDict
|
| 280 |
+
def _moduledict_to_dict(self, module):
|
| 281 |
+
return {key: module for key, module in module.items()}
|
| 282 |
+
|
| 283 |
+
def _forward(self, x: torch.Tensor, mod: torch.Tensor, context: torch.Tensor, mode="default"):
|
| 284 |
+
if self.share_mod:
|
| 285 |
+
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mod.chunk(
|
| 286 |
+
6, dim=1
|
| 287 |
+
)
|
| 288 |
+
else:
|
| 289 |
+
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
|
| 290 |
+
self.adaLN_modulation(mod).chunk(6, dim=1)
|
| 291 |
+
)
|
| 292 |
+
h = _pytree.tree_map(self._apply_module, x, self._moduledict_to_dict(self.norm1))
|
| 293 |
+
h = _pytree.tree_map(
|
| 294 |
+
partial(self._apply_msa, scale_msa=scale_msa, shift_msa=shift_msa),
|
| 295 |
+
h
|
| 296 |
+
)
|
| 297 |
+
h = self.self_attn(h, mode=mode)
|
| 298 |
+
h = _pytree.tree_map(
|
| 299 |
+
partial(self._apply_multiplication, multiplier=gate_msa),
|
| 300 |
+
h
|
| 301 |
+
)
|
| 302 |
+
x = _pytree.tree_map(
|
| 303 |
+
self._apply_add,
|
| 304 |
+
x,
|
| 305 |
+
h
|
| 306 |
+
)
|
| 307 |
+
h = _pytree.tree_map(self._apply_module, x, self._moduledict_to_dict(self.norm2))
|
| 308 |
+
h = _pytree.tree_map(
|
| 309 |
+
partial(self._apply_cross_attn, context=context),
|
| 310 |
+
h,
|
| 311 |
+
self._moduledict_to_dict(self.cross_attn),
|
| 312 |
+
)
|
| 313 |
+
x = _pytree.tree_map(
|
| 314 |
+
self._apply_add,
|
| 315 |
+
x,
|
| 316 |
+
h
|
| 317 |
+
)
|
| 318 |
+
h = _pytree.tree_map(self._apply_module, x, self._moduledict_to_dict(self.norm3))
|
| 319 |
+
h = _pytree.tree_map(
|
| 320 |
+
partial(self._apply_mlp, scale_mlp=scale_mlp, shift_mlp=shift_mlp),
|
| 321 |
+
h
|
| 322 |
+
)
|
| 323 |
+
h = _pytree.tree_map(self._apply_module, h, self._moduledict_to_dict(self.mlp))
|
| 324 |
+
h = _pytree.tree_map(
|
| 325 |
+
partial(self._apply_multiplication, multiplier=gate_mlp),
|
| 326 |
+
h
|
| 327 |
+
)
|
| 328 |
+
x = _pytree.tree_map(
|
| 329 |
+
self._apply_add,
|
| 330 |
+
x,
|
| 331 |
+
h
|
| 332 |
+
)
|
| 333 |
+
return x
|
| 334 |
+
|
| 335 |
+
def forward(self, x: Dict, mod: torch.Tensor, context: torch.Tensor, mode="default"):
|
| 336 |
+
if self.use_checkpoint:
|
| 337 |
+
return torch.utils.checkpoint.checkpoint(
|
| 338 |
+
self._forward, x, mod, context, mode, use_reentrant=False
|
| 339 |
+
)
|
| 340 |
+
else:
|
| 341 |
+
return self._forward(x, mod, context, mode=mode)
|
migrator/code/slat_dev_20261111/sam3d_objects/model/backbone/tdfy_dit/modules/utils.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
import torch
|
| 4 |
+
|
| 5 |
+
FP16_TYPE = torch.float16
|
| 6 |
+
|
| 7 |
+
FP16_MODULES = [
|
| 8 |
+
nn.Conv1d,
|
| 9 |
+
nn.Conv2d,
|
| 10 |
+
nn.Conv3d,
|
| 11 |
+
nn.ConvTranspose1d,
|
| 12 |
+
nn.ConvTranspose2d,
|
| 13 |
+
nn.ConvTranspose3d,
|
| 14 |
+
nn.Linear,
|
| 15 |
+
]
|
| 16 |
+
|
| 17 |
+
# If we add sparse modules back in, they are compatible with FP16.
|
| 18 |
+
# But for now we don't have them and avoid the dependency on FlashAttention
|
| 19 |
+
# Instead using the torch implementation of FlashAttention in SDPA.
|
| 20 |
+
try:
|
| 21 |
+
from ..modules import sparse as sp
|
| 22 |
+
|
| 23 |
+
FP16_MODULES += [
|
| 24 |
+
sp.SparseConv3d,
|
| 25 |
+
sp.SparseInverseConv3d,
|
| 26 |
+
sp.SparseLinear,
|
| 27 |
+
]
|
| 28 |
+
except ImportError:
|
| 29 |
+
pass
|
| 30 |
+
|
| 31 |
+
FP16_MODULES = tuple(FP16_MODULES)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def convert_module_to_f16(l):
|
| 35 |
+
"""
|
| 36 |
+
Convert primitive modules to float16.
|
| 37 |
+
"""
|
| 38 |
+
if isinstance(l, FP16_MODULES):
|
| 39 |
+
for p in l.parameters():
|
| 40 |
+
# p.data = p.data.half()
|
| 41 |
+
p.data = p.data.to(FP16_TYPE)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def convert_module_to_f32(l):
|
| 45 |
+
"""
|
| 46 |
+
Convert primitive modules to float32, undoing convert_module_to_f16().
|
| 47 |
+
"""
|
| 48 |
+
if isinstance(l, FP16_MODULES):
|
| 49 |
+
for p in l.parameters():
|
| 50 |
+
p.data = p.data.float()
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def zero_module(module):
|
| 54 |
+
"""
|
| 55 |
+
Zero out the parameters of a module and return it.
|
| 56 |
+
"""
|
| 57 |
+
for p in module.parameters():
|
| 58 |
+
p.detach().zero_()
|
| 59 |
+
return module
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def scale_module(module, scale):
|
| 63 |
+
"""
|
| 64 |
+
Scale the parameters of a module and return it.
|
| 65 |
+
"""
|
| 66 |
+
for p in module.parameters():
|
| 67 |
+
p.detach().mul_(scale)
|
| 68 |
+
return module
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def modulate(x, shift, scale):
|
| 72 |
+
return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1)
|
migrator/code/slat_dev_20261111/sam3d_objects/model/io.py
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
from typing import Any, Callable, Dict, List, Optional, Union, Iterable
|
| 3 |
+
import lightning.pytorch as pl
|
| 4 |
+
import torch
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
import os
|
| 7 |
+
import re
|
| 8 |
+
from loguru import logger
|
| 9 |
+
from lightning.pytorch.utilities.consolidate_checkpoint import (
|
| 10 |
+
_format_checkpoint,
|
| 11 |
+
_load_distributed_checkpoint,
|
| 12 |
+
)
|
| 13 |
+
from glob import glob
|
| 14 |
+
|
| 15 |
+
from sam3d_objects.data.utils import get_child, set_child
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def rename_checkpoint_weights_using_suffix_matching(
|
| 19 |
+
checkpoint_path_in,
|
| 20 |
+
checkpoint_path_out,
|
| 21 |
+
model: torch.nn.Module,
|
| 22 |
+
strict: bool = True,
|
| 23 |
+
keys: Optional[List[Any]] = (),
|
| 24 |
+
):
|
| 25 |
+
# extract model names
|
| 26 |
+
param_names = [n for n, _ in model.named_parameters()]
|
| 27 |
+
buffer_names = [n for n, _ in model.named_buffers()]
|
| 28 |
+
model_names = param_names + buffer_names
|
| 29 |
+
|
| 30 |
+
# load stored weights
|
| 31 |
+
state = torch.load(checkpoint_path_in, weights_only=False)
|
| 32 |
+
|
| 33 |
+
model_state = get_child(state, *keys)
|
| 34 |
+
model_state_names = list(model_state.keys())
|
| 35 |
+
|
| 36 |
+
# sort reversed names (sort by suffix)
|
| 37 |
+
model_names_rev = sorted([n[::-1] for n in model_names])
|
| 38 |
+
model_state_names_rev = sorted([n[::-1] for n in model_state_names])
|
| 39 |
+
|
| 40 |
+
if strict and len(model_names) != len(model_state_names):
|
| 41 |
+
raise RuntimeError(
|
| 42 |
+
f"model and state don't have the same number of parameters ({len(model_names)} != {len(model_state_names)}), cannot match them (set strict = False to relax constraint)"
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
def common_prefix_length(str_0: str, str_1: str):
|
| 46 |
+
for count in range(min(len(str_0), len(str_1))):
|
| 47 |
+
if str_0[count] != str_1[count]:
|
| 48 |
+
break
|
| 49 |
+
return count
|
| 50 |
+
|
| 51 |
+
# attempt to match every model names to largest suffic matched weight
|
| 52 |
+
name_mapping = {}
|
| 53 |
+
i, j = 0, 0
|
| 54 |
+
last_n = 0
|
| 55 |
+
while i < len(model_names_rev):
|
| 56 |
+
if j < len(model_state_names_rev):
|
| 57 |
+
n = common_prefix_length(model_names_rev[i], model_state_names_rev[j])
|
| 58 |
+
else:
|
| 59 |
+
n = 0
|
| 60 |
+
|
| 61 |
+
if n >= last_n:
|
| 62 |
+
last_n = n
|
| 63 |
+
j += 1
|
| 64 |
+
else:
|
| 65 |
+
last_n = 0
|
| 66 |
+
name_mapping[model_names_rev[i][::-1]] = model_state_names_rev[j - 1][::-1]
|
| 67 |
+
i += 1
|
| 68 |
+
|
| 69 |
+
if not j < len(model_state_names_rev) + 1:
|
| 70 |
+
break
|
| 71 |
+
|
| 72 |
+
# not all names might have been matched
|
| 73 |
+
if i < len(model_names):
|
| 74 |
+
raise RuntimeError("could not suffix match parameter names")
|
| 75 |
+
|
| 76 |
+
for k, v in name_mapping.items():
|
| 77 |
+
logger.debug(f"{k} <- {v}")
|
| 78 |
+
|
| 79 |
+
# rename weights according to matches and save to disk
|
| 80 |
+
model_state_out = {k: model_state[v] for k, v in name_mapping.items()}
|
| 81 |
+
set_child(state, model_state_out, *keys)
|
| 82 |
+
torch.save(state, checkpoint_path_out)
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def remove_prefix_state_dict_fn(prefix: str):
|
| 86 |
+
n = len(prefix)
|
| 87 |
+
|
| 88 |
+
def state_dict_fn(state_dict):
|
| 89 |
+
return {
|
| 90 |
+
(key[n:] if key.startswith(prefix) else key): value
|
| 91 |
+
for key, value in state_dict.items()
|
| 92 |
+
}
|
| 93 |
+
|
| 94 |
+
return state_dict_fn
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def add_prefix_state_dict_fn(prefix: str):
|
| 98 |
+
def state_dict_fn(state_dict):
|
| 99 |
+
return {prefix + key: value for key, value in state_dict.items()}
|
| 100 |
+
|
| 101 |
+
return state_dict_fn
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def filter_and_remove_prefix_state_dict_fn(prefix: str):
|
| 105 |
+
n = len(prefix)
|
| 106 |
+
|
| 107 |
+
def state_dict_fn(state_dict):
|
| 108 |
+
return {
|
| 109 |
+
key[n:]: value
|
| 110 |
+
for key, value in state_dict.items()
|
| 111 |
+
if key.startswith(prefix)
|
| 112 |
+
}
|
| 113 |
+
|
| 114 |
+
return state_dict_fn
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def get_last_checkpoint(path: str):
|
| 118 |
+
checkpoints = glob(os.path.join(path, "epoch=*-step=*.ckpt"))
|
| 119 |
+
prog = re.compile(r"epoch=(\d+)-step=(\d+).ckpt")
|
| 120 |
+
|
| 121 |
+
checkpoints_to_sort = []
|
| 122 |
+
for checkpoint in checkpoints:
|
| 123 |
+
checkpoint_name = os.path.basename(checkpoint)
|
| 124 |
+
match = prog.match(checkpoint_name)
|
| 125 |
+
if match is not None:
|
| 126 |
+
n_epoch, n_step = prog.match(checkpoint_name).groups()
|
| 127 |
+
n_epoch, n_step = int(n_epoch), int(n_step)
|
| 128 |
+
checkpoints_to_sort.append((n_epoch, n_step, checkpoint))
|
| 129 |
+
|
| 130 |
+
sorted_checkpoints = sorted(checkpoints_to_sort)
|
| 131 |
+
if not len(sorted_checkpoints) > 0:
|
| 132 |
+
raise RuntimeError(f"no checkpoint has been found at path : {path}")
|
| 133 |
+
return sorted_checkpoints[-1][2]
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def load_sharded_checkpoint(path: str, device: Optional[str]):
|
| 137 |
+
if device != "cpu":
|
| 138 |
+
raise RuntimeError(
|
| 139 |
+
f'loading sharded weights on device "{device}" is not available, please use the "cpu" device instead'
|
| 140 |
+
)
|
| 141 |
+
checkpoint = _load_distributed_checkpoint(Path(path))
|
| 142 |
+
checkpoint = _format_checkpoint(checkpoint)
|
| 143 |
+
return checkpoint
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def load_model_from_checkpoint(
|
| 147 |
+
model: Union[pl.LightningModule, torch.nn.Module],
|
| 148 |
+
checkpoint_path: str,
|
| 149 |
+
strict: bool = True,
|
| 150 |
+
device: Optional[str] = None,
|
| 151 |
+
freeze: bool = False,
|
| 152 |
+
eval: bool = False,
|
| 153 |
+
map_name: Union[Dict[str, str], None] = None,
|
| 154 |
+
remove_name: Union[List[str], None] = None,
|
| 155 |
+
state_dict_key: Union[None, str, Iterable[str]] = "state_dict",
|
| 156 |
+
state_dict_fn: Optional[Callable[[Any], Any]] = None,
|
| 157 |
+
):
|
| 158 |
+
logger.info(f"Loading checkpoint from {checkpoint_path}")
|
| 159 |
+
if os.path.isfile(checkpoint_path):
|
| 160 |
+
checkpoint = torch.load(
|
| 161 |
+
checkpoint_path,
|
| 162 |
+
map_location=device,
|
| 163 |
+
weights_only=False,
|
| 164 |
+
)
|
| 165 |
+
elif os.path.isdir(checkpoint_path): # sharded
|
| 166 |
+
checkpoint = load_sharded_checkpoint(checkpoint_path, device=device)
|
| 167 |
+
else: # if neither a file nor a directory, path does not exist
|
| 168 |
+
raise FileNotFoundError(checkpoint_path)
|
| 169 |
+
|
| 170 |
+
if isinstance(model, pl.LightningModule):
|
| 171 |
+
model.on_load_checkpoint(checkpoint)
|
| 172 |
+
|
| 173 |
+
# get state dictionary
|
| 174 |
+
state_dict = checkpoint
|
| 175 |
+
if state_dict_key is not None:
|
| 176 |
+
if isinstance(state_dict_key, str):
|
| 177 |
+
state_dict_key = (state_dict_key,)
|
| 178 |
+
state_dict = get_child(state_dict, *state_dict_key)
|
| 179 |
+
|
| 180 |
+
# remove names
|
| 181 |
+
if remove_name is not None:
|
| 182 |
+
for name in remove_name:
|
| 183 |
+
del state_dict[name]
|
| 184 |
+
|
| 185 |
+
# remap names
|
| 186 |
+
if map_name is not None:
|
| 187 |
+
for src, dst in map_name.items():
|
| 188 |
+
if src not in state_dict:
|
| 189 |
+
continue
|
| 190 |
+
state_dict[dst] = state_dict[src]
|
| 191 |
+
del state_dict[src]
|
| 192 |
+
|
| 193 |
+
# apply custom changes to dict
|
| 194 |
+
if state_dict_fn is not None:
|
| 195 |
+
state_dict = state_dict_fn(state_dict)
|
| 196 |
+
|
| 197 |
+
model.load_state_dict(state_dict, strict=strict)
|
| 198 |
+
|
| 199 |
+
if device is not None:
|
| 200 |
+
model = model.to(device)
|
| 201 |
+
|
| 202 |
+
if freeze:
|
| 203 |
+
for param in model.parameters():
|
| 204 |
+
param.requires_grad = False
|
| 205 |
+
eval = True
|
| 206 |
+
|
| 207 |
+
if eval:
|
| 208 |
+
model.eval()
|
| 209 |
+
|
| 210 |
+
return model
|
migrator/code/slat_dev_20261111/sam3d_objects/model/layers/__init__.py
ADDED
|
File without changes
|
migrator/code/slat_dev_20261111/sam3d_objects/model/layers/llama3/__init__.py
ADDED
|
File without changes
|
migrator/code/slat_dev_20261111/sam3d_objects/model/layers/llama3/ff.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
from torch import nn
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
from typing import Optional
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class FeedForward(nn.Module):
|
| 8 |
+
def __init__(
|
| 9 |
+
self,
|
| 10 |
+
dim: int,
|
| 11 |
+
hidden_dim: int,
|
| 12 |
+
multiple_of: int = 256,
|
| 13 |
+
ffn_dim_multiplier: Optional[float] = None,
|
| 14 |
+
output_dim: Optional[int] = None,
|
| 15 |
+
skip_w2: bool = False,
|
| 16 |
+
):
|
| 17 |
+
"""
|
| 18 |
+
Llama3 FeedForward layer
|
| 19 |
+
https://github.com/meta-llama/llama3/blob/a0940f9cf7065d45bb6675660f80d305c041a754/llama/model.py#L193
|
| 20 |
+
"""
|
| 21 |
+
super().__init__()
|
| 22 |
+
hidden_dim = int(2 * hidden_dim / 3)
|
| 23 |
+
# custom dim factor multiplier
|
| 24 |
+
if ffn_dim_multiplier is not None:
|
| 25 |
+
hidden_dim = int(ffn_dim_multiplier * hidden_dim)
|
| 26 |
+
hidden_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of)
|
| 27 |
+
|
| 28 |
+
if output_dim is None:
|
| 29 |
+
output_dim = dim
|
| 30 |
+
|
| 31 |
+
self.skip_w2 = skip_w2
|
| 32 |
+
self.w1 = nn.Linear(dim, hidden_dim, bias=False)
|
| 33 |
+
if not self.skip_w2:
|
| 34 |
+
self.w2 = nn.Linear(hidden_dim, output_dim, bias=False)
|
| 35 |
+
self.w3 = nn.Linear(dim, hidden_dim, bias=False)
|
| 36 |
+
|
| 37 |
+
def forward(self, x):
|
| 38 |
+
x = F.silu(self.w1(x)) * self.w3(x)
|
| 39 |
+
if self.skip_w2:
|
| 40 |
+
return x
|
| 41 |
+
return self.w2(x)
|