| import torch
|
| from torch import nn
|
| import timm
|
| import math
|
|
|
|
|
| '''
|
| def forward_block(self, x):
|
| x = x + self.drop_path(self.attn(self.norm1(x))) + self.drop_path(self.adapter_attn(self.norm1(x))) * self.s
|
| x = x + self.drop_path(self.mlp(self.norm2(x))) + self.drop_path(self.adapter_mlp(self.norm2(x))) * self.s
|
| return x
|
|
|
|
|
| def forward_block_attn(self, x):
|
| x = x + self.drop_path(self.attn(self.norm1(x))) + self.drop_path(self.adapter_attn(self.norm1(x))) * self.s
|
| x = x + self.drop_path(self.mlp(self.norm2(x)))
|
| return x
|
| '''
|
|
|
|
|
| class QuickGELU(nn.Module):
|
| def forward(self, x: torch.Tensor):
|
| return x * torch.sigmoid(1.702 * x)
|
|
|
|
|
|
|
| class Bi_direct_adapter(nn.Module):
|
| def __init__(self, dim=8,input_dim=768, xavier_init=False):
|
| super().__init__()
|
|
|
| self.adapter_down = nn.Linear(input_dim, dim)
|
| self.adapter_up = nn.Linear(dim, input_dim)
|
| self.adapter_mid = nn.Linear(dim, dim)
|
|
|
|
|
| nn.init.zeros_(self.adapter_mid.bias)
|
| nn.init.zeros_(self.adapter_mid.weight)
|
| nn.init.zeros_(self.adapter_down.weight)
|
| nn.init.zeros_(self.adapter_down.bias)
|
| nn.init.zeros_(self.adapter_up.weight)
|
| nn.init.zeros_(self.adapter_up.bias)
|
|
|
|
|
| self.dropout = nn.Dropout(0.1)
|
| self.dim = dim
|
|
|
| def forward(self, x):
|
|
|
| x_down = self.adapter_down(x)
|
|
|
| x_down = self.adapter_mid(x_down)
|
|
|
| x_down = self.dropout(x_down)
|
| x_up = self.adapter_up(x_down)
|
|
|
| return x_up
|
|
|
| """
|
|
|
|
|
| class Convpass(nn.Module):
|
| def __init__(self, dim=8, xavier_init=False):
|
| super().__init__()
|
|
|
| self.adapter_conv = nn.Conv2d(dim, dim, 3, 1, 1)
|
| if xavier_init:
|
| nn.init.xavier_uniform_(self.adapter_conv.weight)
|
| else:
|
| nn.init.zeros_(self.adapter_conv.weight)
|
| self.adapter_conv.weight.data[:, :, 1, 1] += torch.eye(8, dtype=torch.float)
|
| nn.init.zeros_(self.adapter_conv.bias)
|
|
|
| self.adapter_down = nn.Linear(768, dim) # equivalent to 1 * 1 Conv
|
| self.adapter_up = nn.Linear(dim, 768) # equivalent to 1 * 1 Conv
|
| nn.init.xavier_uniform_(self.adapter_down.weight)
|
| nn.init.zeros_(self.adapter_down.bias)
|
| nn.init.zeros_(self.adapter_up.weight)
|
| nn.init.zeros_(self.adapter_up.bias)
|
|
|
| self.act = QuickGELU()
|
| self.dropout = nn.Dropout(0.1)
|
| self.dim = dim
|
|
|
| def forward(self, x):
|
| B, N, C = x.shape
|
| #print(x.shape)
|
| x_down = self.adapter_down(x) # equivalent to 1 * 1 Conv
|
| x_down = self.act(x_down)
|
|
|
| #print(x_down.shape)
|
|
|
| x_patch = x_down[:, 64:].reshape(B, 16, 16, self.dim).permute(0, 3, 1, 2) ############
|
| x_patch = self.adapter_conv(x_patch)
|
| x_patch = x_patch.permute(0, 2, 3, 1).reshape(B, 16 * 16, self.dim)
|
|
|
|
|
| #x_down = torch.cat([x_cls, x_patch], dim=1)
|
|
|
| x_down = self.act(x_down)
|
| x_down = self.dropout(x_down)
|
| x_up = self.adapter_up(x_down) # equivalent to 1 * 1 Conv
|
|
|
| return x_up
|
| """ |
