Spaces:
Running on Zero
Running on Zero
Download diffcve/modules/attention.py from hugging-apps/diffcve-2607: direct link, hf CLI and curl.
- Browser
- Download file 15.2 kB
-
https://huggingface.co/spaces/hugging-apps/diffcve-2607/resolve/main/diffcve/modules/attention.py
- Command line
-
hf download hf://spaces/hugging-apps/diffcve-2607/diffcve/modules/attention.py
-
curl -L -o attention.py https://huggingface.co/spaces/hugging-apps/diffcve-2607/resolve/main/diffcve/modules/attention.py
15.2 kB
| from inspect import isfunction | |
| import torch | |
| import torch.nn.functional as F | |
| from torch import nn, einsum | |
| from einops import rearrange, repeat | |
| from diffcve.modules.diffusion.utils import checkpoint | |
| from diffcve.modules.diffusion.utils import from_4d_to_3d, from_3d_to_4d | |
| try: | |
| import xformers | |
| import xformers.ops | |
| XFORMERS_AVAILABLE = True | |
| except: | |
| XFORMERS_AVAILABLE = False | |
| def exists(val): | |
| return val is not None | |
| def default(val, d): | |
| if exists(val): | |
| return val | |
| return d() if isfunction(d) else d | |
| def max_neg_value(t): | |
| return -torch.finfo(t.dtype).max | |
| class GEGLU(nn.Module): | |
| def __init__(self, dim_in, dim_out): | |
| super().__init__() | |
| self.proj = nn.Linear(dim_in, dim_out * 2) | |
| def forward(self, x): | |
| x, gate = self.proj(x).chunk(2, dim=-1) | |
| return x * F.gelu(gate) | |
| class FeedForward(nn.Module): | |
| def __init__(self, dim, dim_out=None, mult=4, glu=False, dropout=0.): | |
| super().__init__() | |
| inner_dim = int(dim * mult) | |
| dim_out = default(dim_out, dim) | |
| project_in = nn.Sequential( | |
| nn.Linear(dim, inner_dim), | |
| nn.GELU() | |
| ) if not glu else GEGLU(dim, inner_dim) | |
| self.net = nn.Sequential( | |
| project_in, | |
| nn.Dropout(dropout), | |
| nn.Linear(inner_dim, dim_out) | |
| ) | |
| def forward(self, x): | |
| return self.net(x) | |
| def zero_module(module): | |
| for p in module.parameters(): | |
| p.detach().zero_() | |
| return module | |
| def Normalize(in_channels): | |
| return torch.nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) | |
| class TemporalAttention(nn.Module): | |
| def __init__(self, num_feat, num_heads=8, dim_head=64, num_frames=1): | |
| super().__init__() | |
| self.num_frames = num_frames | |
| self.temporal_attn = MemoryEfficientSelfAttention(num_feat, heads=num_heads, dim_head=dim_head, dropout=0.0) | |
| self.norm = nn.LayerNorm(num_feat) | |
| self.temporal_alpha = nn.Parameter(torch.Tensor(1)) | |
| def forward(self, inp, t=None): | |
| bt, c, h, w = inp.shape | |
| # einops >= 0.4 raises when an explicit axis length is None (the authors' | |
| # einops 0.3.0 inferred it from the shape); use the effective frame count. | |
| t = t or self.num_frames | |
| b = bt // t | |
| ori = inp | |
| inp = from_4d_to_3d(inp, b, c, t, h, w) | |
| res = self.temporal_attn(self.norm(inp)) | |
| res = from_3d_to_4d(res, b, c, t, h, w) | |
| out = self.temporal_alpha * res + (1 - self.temporal_alpha) * ori | |
| return out | |
| class LinearAttention(nn.Module): | |
| def __init__(self, dim, heads=4, dim_head=32): | |
| super().__init__() | |
| self.heads = heads | |
| hidden_dim = dim_head * heads | |
| self.to_qkv = nn.Conv2d(dim, hidden_dim * 3, 1, bias = False) | |
| self.to_out = nn.Conv2d(hidden_dim, dim, 1) | |
| def forward(self, x): | |
| b, c, h, w = x.shape | |
| qkv = self.to_qkv(x) | |
| q, k, v = rearrange(qkv, 'b (qkv heads c) h w -> qkv b heads c (h w)', heads = self.heads, qkv=3) | |
| k = k.softmax(dim=-1) | |
| context = torch.einsum('bhdn,bhen->bhde', k, v) | |
| out = torch.einsum('bhde,bhdn->bhen', context, q) | |
| out = rearrange(out, 'b heads c (h w) -> b (heads c) h w', heads=self.heads, h=h, w=w) | |
| return self.to_out(out) | |
| class CrossAttention(nn.Module): | |
| def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0.): | |
| super().__init__() | |
| inner_dim = dim_head * heads | |
| context_dim = default(context_dim, query_dim) | |
| self.scale = dim_head ** -0.5 | |
| self.heads = heads | |
| self.to_q = nn.Linear(query_dim, inner_dim, bias=False) | |
| self.to_k = nn.Linear(context_dim, inner_dim, bias=False) | |
| self.to_v = nn.Linear(context_dim, inner_dim, bias=False) | |
| self.to_out = nn.Sequential( | |
| nn.Linear(inner_dim, query_dim), | |
| nn.Dropout(dropout) | |
| ) | |
| def forward(self, x, context=None, mask=None): | |
| h = self.heads | |
| q = self.to_q(x) | |
| context = default(context, x) | |
| k = self.to_k(context) | |
| v = self.to_v(context) | |
| q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> (b h) n d', h=h), (q, k, v)) | |
| sim = einsum('b i d, b j d -> b i j', q, k) * self.scale | |
| if exists(mask): | |
| mask = rearrange(mask, 'b ... -> b (...)') | |
| max_neg_value = -torch.finfo(sim.dtype).max | |
| mask = repeat(mask, 'b j -> (b h) () j', h=h) | |
| sim.masked_fill_(~mask, max_neg_value) | |
| attn = sim.softmax(dim=-1) | |
| out = einsum('b i j, b j d -> b i d', attn, v) | |
| out = rearrange(out, '(b h) n d -> b n (h d)', h=h) | |
| return self.to_out(out) | |
| class MemoryEfficientSelfAttention(nn.Module): | |
| # https://github.com/MatthieuTPHR/diffusers/blob/d80b531ff8060ec1ea982b65a1b8df70f73aa67c/src/diffusers/models/attention.py#L223 | |
| def __init__(self, query_dim, heads=8, dim_head=64, dropout=0.0): | |
| super().__init__() | |
| print(f"Setting up {self.__class__.__name__}. Query dim is {query_dim}, using {heads} heads.") | |
| inner_dim = dim_head * heads | |
| self.heads = heads | |
| self.dim_head = dim_head | |
| self.to_q = nn.Linear(query_dim, inner_dim, bias=False) | |
| self.to_k = nn.Linear(query_dim, inner_dim, bias=False) | |
| self.to_v = nn.Linear(query_dim, inner_dim, bias=False) | |
| self.to_out = nn.Sequential( | |
| nn.Linear(inner_dim, query_dim), | |
| nn.Dropout(dropout) | |
| ) | |
| self.attention_op: Optional[Any] = None | |
| def forward(self, x, mask=None): | |
| q = self.to_q(x) | |
| k = self.to_k(x) | |
| v = self.to_v(x) | |
| b, _, _ = q.shape | |
| q, k, v = map( | |
| lambda t: t.unsqueeze(3) | |
| .reshape(b, t.shape[1], self.heads, self.dim_head) | |
| .permute(0, 2, 1, 3) | |
| .reshape(b * self.heads, t.shape[1], self.dim_head) | |
| .contiguous(), | |
| (q, k, v), | |
| ) | |
| if torch.onnx.is_in_onnx_export(): | |
| scale = self.dim_head ** -0.5 | |
| attention = torch.matmul(q, k.transpose(-1, -2)) * scale | |
| attention = attention.softmax(dim=-1) | |
| out = torch.matmul(attention, v) | |
| else: | |
| out = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=None, op=self.attention_op) | |
| if exists(mask): | |
| raise NotImplementedError | |
| out = ( | |
| out.unsqueeze(0) | |
| .reshape(b, self.heads, out.shape[1], self.dim_head) | |
| .permute(0, 2, 1, 3) | |
| .reshape(b, out.shape[1], self.heads * self.dim_head) | |
| ) | |
| return self.to_out(out) | |
| class MemoryEfficientCrossAttention(nn.Module): | |
| # https://github.com/MatthieuTPHR/diffusers/blob/d80b531ff8060ec1ea982b65a1b8df70f73aa67c/src/diffusers/models/attention.py#L223 | |
| def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0.0): | |
| super().__init__() | |
| print(f"Setting up {self.__class__.__name__}. Query dim is {query_dim}, context_dim is {context_dim} and using " | |
| f"{heads} heads.") | |
| inner_dim = dim_head * heads | |
| context_dim = default(context_dim, query_dim) | |
| self.heads = heads | |
| self.dim_head = dim_head | |
| self.to_q = nn.Linear(query_dim, inner_dim, bias=False) | |
| self.to_k = nn.Linear(context_dim, inner_dim, bias=False) | |
| self.to_v = nn.Linear(context_dim, inner_dim, bias=False) | |
| self.to_out = nn.Sequential( | |
| nn.Linear(inner_dim, query_dim), | |
| nn.Dropout(dropout) | |
| ) | |
| self.attention_op: Optional[Any] = None | |
| def forward(self, x, context=None, mask=None): | |
| q = self.to_q(x) | |
| context = default(context, x) | |
| if x.shape[0] != context.shape[0]: | |
| context = torch.repeat_interleave(context, x.shape[0] // context.shape[0], dim=0) | |
| k = self.to_k(context) | |
| v = self.to_v(context) | |
| b, _, _ = q.shape | |
| q, k, v = map( | |
| lambda t: t.unsqueeze(3) | |
| .reshape(b, t.shape[1], self.heads, self.dim_head) | |
| .permute(0, 2, 1, 3) | |
| .reshape(b * self.heads, t.shape[1], self.dim_head) | |
| .contiguous(), | |
| (q, k, v), | |
| ) | |
| if torch.onnx.is_in_onnx_export(): | |
| scale = self.dim_head ** -0.5 | |
| attention = torch.matmul(q, k.transpose(-1, -2)) * scale | |
| attention = attention.softmax(dim=-1) | |
| out = torch.matmul(attention, v) | |
| else: | |
| out = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=None, op=self.attention_op) | |
| if exists(mask): | |
| raise NotImplementedError | |
| out = ( | |
| out.unsqueeze(0) | |
| .reshape(b, self.heads, out.shape[1], self.dim_head) | |
| .permute(0, 2, 1, 3) | |
| .reshape(b, out.shape[1], self.heads * self.dim_head) | |
| ) | |
| return self.to_out(out) | |
| class BasicTransformerBlock(nn.Module): | |
| def __init__(self, dim, n_heads, d_head, dropout=0., context_dim=None, gated_ff=True, checkpoint=False): | |
| super().__init__() | |
| self.attn1 = CrossAttention(query_dim=dim, heads=n_heads, dim_head=d_head, dropout=dropout) | |
| self.ff = FeedForward(dim, dropout=dropout, glu=gated_ff) | |
| self.attn2 = CrossAttention(query_dim=dim, context_dim=context_dim, | |
| heads=n_heads, dim_head=d_head, dropout=dropout) | |
| self.norm1 = nn.LayerNorm(dim) | |
| self.norm2 = nn.LayerNorm(dim) | |
| self.norm3 = nn.LayerNorm(dim) | |
| self.checkpoint = checkpoint | |
| def forward(self, x, context=None): | |
| return checkpoint(self._forward, (x, context), self.parameters(), self.checkpoint) | |
| def _forward(self, x, context=None): | |
| x = self.attn1(self.norm1(x)) + x | |
| x = self.attn2(self.norm2(x), context=context) + x | |
| x = self.ff(self.norm3(x)) + x | |
| return x | |
| class ConfigurableTransformerBlock(nn.Module): | |
| ATTENTION_MODES = { | |
| "softmax": CrossAttention, | |
| "softmax-xformers": MemoryEfficientCrossAttention | |
| } | |
| def __init__(self, dim, n_heads, d_head, dropout=0., context_dim=None, gated_ff=True, checkpoint=True, | |
| disable_self_attn=False): | |
| super().__init__() | |
| attn_mode = "softmax-xformers" if XFORMERS_AVAILABLE else "softmax" | |
| assert attn_mode in self.ATTENTION_MODES | |
| attn_cls = self.ATTENTION_MODES[attn_mode] | |
| self.disable_self_attn = disable_self_attn | |
| self.attn1 = attn_cls(query_dim=dim, heads=n_heads, dim_head=d_head, dropout=dropout, | |
| context_dim=context_dim if self.disable_self_attn else None) | |
| self.ff = FeedForward(dim, dropout=dropout, glu=gated_ff) | |
| self.attn2 = attn_cls(query_dim=dim, context_dim=context_dim, | |
| heads=n_heads, dim_head=d_head, dropout=dropout) | |
| self.norm1 = nn.LayerNorm(dim) | |
| self.norm2 = nn.LayerNorm(dim) | |
| self.norm3 = nn.LayerNorm(dim) | |
| self.checkpoint = checkpoint | |
| def forward(self, x, context=None): | |
| return checkpoint(self._forward, (x, context), self.parameters(), self.checkpoint) | |
| def _forward(self, x, context=None): | |
| x = self.attn1(self.norm1(x), context=context if self.disable_self_attn else None) + x | |
| x = self.attn2(self.norm2(x), context=context) + x | |
| x = self.ff(self.norm3(x)) + x | |
| return x | |
| class SpatialTransformer(nn.Module): | |
| def __init__(self, in_channels, n_heads, d_head, | |
| depth=1, dropout=0., context_dim=None): | |
| super().__init__() | |
| self.in_channels = in_channels | |
| inner_dim = n_heads * d_head | |
| self.norm = Normalize(in_channels) | |
| self.proj_in = nn.Conv2d(in_channels, | |
| inner_dim, | |
| kernel_size=1, | |
| stride=1, | |
| padding=0) | |
| self.transformer_blocks = nn.ModuleList( | |
| [BasicTransformerBlock(inner_dim, n_heads, d_head, dropout=dropout, context_dim=context_dim) | |
| for d in range(depth)] | |
| ) | |
| self.proj_out = zero_module(nn.Conv2d(inner_dim, | |
| in_channels, | |
| kernel_size=1, | |
| stride=1, | |
| padding=0)) | |
| def forward(self, x, context=None): | |
| b, c, h, w = x.shape | |
| x_in = x | |
| x = self.norm(x) | |
| x = self.proj_in(x) | |
| x = rearrange(x, 'b c h w -> b (h w) c') | |
| for block in self.transformer_blocks: | |
| x = block(x, context=context) | |
| x = rearrange(x, 'b (h w) c -> b c h w', h=h, w=w) | |
| x = self.proj_out(x) | |
| return x + x_in | |
| class ProjectedSpatialTransformer(nn.Module): | |
| def __init__(self, in_channels, n_heads, d_head, | |
| depth=1, dropout=0., context_dim=None, | |
| disable_self_attn=False, use_linear=False, | |
| use_checkpoint=False): | |
| super().__init__() | |
| if exists(context_dim) and not isinstance(context_dim, list): | |
| context_dim = [context_dim] | |
| self.in_channels = in_channels | |
| inner_dim = n_heads * d_head | |
| self.norm = Normalize(in_channels) | |
| if not use_linear: | |
| self.proj_in = nn.Conv2d(in_channels, | |
| inner_dim, | |
| kernel_size=1, | |
| stride=1, | |
| padding=0) | |
| else: | |
| self.proj_in = nn.Linear(in_channels, inner_dim) | |
| self.transformer_blocks = nn.ModuleList( | |
| [ConfigurableTransformerBlock(inner_dim, n_heads, d_head, dropout=dropout, context_dim=context_dim[d], | |
| disable_self_attn=disable_self_attn, checkpoint=use_checkpoint) | |
| for d in range(depth)] | |
| ) | |
| if not use_linear: | |
| self.proj_out = zero_module(nn.Conv2d(inner_dim, | |
| in_channels, | |
| kernel_size=1, | |
| stride=1, | |
| padding=0)) | |
| else: | |
| self.proj_out = zero_module(nn.Linear(in_channels, inner_dim)) | |
| self.use_linear = use_linear | |
| def forward(self, x, context=None): | |
| if not isinstance(context, list): | |
| context = [context] | |
| b, c, h, w = x.shape | |
| x_in = x | |
| x = self.norm(x) | |
| if not self.use_linear: | |
| x = self.proj_in(x) | |
| x = rearrange(x, 'b c h w -> b (h w) c').contiguous() | |
| if self.use_linear: | |
| x = self.proj_in(x) | |
| for i, block in enumerate(self.transformer_blocks): | |
| x = block(x, context=context[i]) | |
| if self.use_linear: | |
| x = self.proj_out(x) | |
| x = rearrange(x, 'b (h w) c -> b c h w', h=h, w=w).contiguous() | |
| if not self.use_linear: | |
| x = self.proj_out(x) | |
| return x + x_in | |