import torch from torch import nn from torch.nn import functional as F class DSMIL_Attention(nn.Module): def __init__(self): super().__init__() # q(patch_num, size[2]), q_max(num_classes, size[2]) def forward(self, q, q_max): attn = q @ q_max.transpose(1, 0) # (patch_num, num_classes) return F.softmax(attn / torch.sqrt(torch.tensor(q.shape[1], dtype=torch.float32)), dim=0) # (patch_num, num_classes) class DSMIL_BClassifier(nn.Module): def __init__(self, num_classes: int, size = [768, 128, 128], dropout: float = 0.5, ): super().__init__() self.q = nn.Sequential( nn.Linear(size[0], size[1]), nn.ReLU(), nn.Linear(size[1], size[2]), nn.Tanh() ) self.v = nn.Sequential( nn.Dropout(dropout), nn.Linear(size[0], size[0]), nn.ReLU() ) self.attention = DSMIL_Attention() self.classifier = nn.Sequential( nn.Dropout(dropout), nn.Conv1d(num_classes, num_classes, kernel_size=size[0]) ) # x(patch_num, size[0]), inst_logits(patch_num, num_classes) def forward(self, x, inst_logits): v = self.v(x) # (patch_num, size[0]) q = self.q(x) # (patch_num, size[2]) _, idxs = torch.sort(inst_logits, dim=0, descending=True) # (patch_num, num_classes) idxs = idxs[0] # (num_classes,) x_sub = x[idxs] # (num_classes, size[0]) q_max = self.q(x_sub) # (num_classes, size[2]) attn = self.attention(q, q_max) # (patch_num, num_classes) bag_feature = attn.transpose(1, 0) @ v # (num_classes, size[0]) bag_logits = self.classifier(bag_feature)[:, 0] # (num_classes,) return bag_logits, attn, bag_feature class DSMIL(nn.Module): def __init__(self, num_classes: int, size = [768, 128, 128], dropout: float = 0.5, ): super().__init__() self.i_classifier = nn.Linear(size[0], num_classes) self.b_classifier = DSMIL_BClassifier(num_classes, size, dropout) def forward(self, x): inst_logits = self.i_classifier(x) bag_logits, attn, bag_feature = self.b_classifier(x, inst_logits) # (num_classes,), (N, num_classes), return bag_logits, inst_logits, attn, bag_feature