import torch from torch import nn import torch.nn.functional as F from math import cos, pi, sin import math import numpy as np from scipy.special import lambertw def mixup_criterion(criterion, pred, y_a, y_b, lam, pow=2): y = lam ** pow * y_a + (1 - lam) ** pow * y_b return criterion(pred, y) def mixup_data(v, q, a): '''Returns mixed inputs, pairs of targets, and lambda without organ constraint''' lam = np.random.beta(1, 1) batch_size = v.shape[0] index = torch.randperm(batch_size) mixed_v = lam * v + (1 - lam) * v[index, :] mixed_q = lam * q + (1 - lam) * q[index, :] a_1, a_2 = a, a[index] return mixed_v, mixed_q, a_1, a_2, lam def linear(epoch, nepoch): return 1 - epoch / nepoch def convex(epoch, nepoch): return epoch / (2 - nepoch) def concave(epoch, nepoch): return 1 - sin((epoch / nepoch) * (pi / 2)) def composite(epoch, nepoch): return 0.5 * cos((epoch / nepoch) * pi) + 0.5 class LogCoshLoss(nn.Module): def __init__(self): super().__init__() def forward(self, y_t, y_prime_t): ey_t = y_t - y_prime_t return torch.mean(torch.log(torch.cosh(ey_t + 1e-12)))+F.mse_loss(y_t, y_prime_t) class MLCE(nn.Module): def __init__(self): super(MLCE, self).__init__() def _mlcce(self, y_pred, y_true): y_pred = (1 - 2 * y_true) * y_pred y_pred_neg = y_pred - y_true * 1e12 y_pred_pos = y_pred - (1 - y_true) * 1e12 zeros = torch.zeros_like(y_pred[..., :1]) y_pred_neg = torch.cat([y_pred_neg, zeros], dim=-1) y_pred_pos = torch.cat([y_pred_pos, zeros], dim=-1) neg_loss = torch.logsumexp(y_pred_neg, dim=-1) pos_loss = torch.logsumexp(y_pred_pos, dim=-1) loss = torch.mean(neg_loss + pos_loss) return loss def __call__(self, y_pred, y_true): return self._mlcce(y_pred, y_true) class SuperLoss(nn.Module): def __init__(self, C=10, lam=1, batch_size=128): super(SuperLoss, self).__init__() self.tau = math.log(C) self.lam = lam # set to 1 for CIFAR10 and 0.25 for CIFAR100 self.batch_size = batch_size def forward(self, logits, targets): l_i = F.mse_loss(logits, targets, reduction='none').detach() sigma = self.sigma(l_i) loss = (F.mse_loss(logits, targets, reduction='none') - self.tau) * sigma + self.lam * ( torch.log(sigma) ** 2) loss = loss.sum() / self.batch_size return loss def sigma(self, l_i): x = torch.ones(l_i.size()) * (-2 / math.exp(1.)) x = x.cuda() y = 0.5 * torch.max(x, (l_i - self.tau) / self.lam) y = y.cpu().numpy() sigma = np.exp(-lambertw(y)) sigma = sigma.real.astype(np.float32) sigma = torch.from_numpy(sigma).cuda() return sigma def unbiased_curriculum_loss(out, data, args, epoch, epochs, scheduler='linear'): losses = [] scheduler = linear if scheduler == 'linear' else concave # calculate difficulty measurement function adjusted_losses = [] for idx in range(out.shape[0]): ground_truth = max(1, abs(data[idx].item())) loss = F.mse_loss(out[idx], data[idx]) losses.append(loss) adjusted_losses.append(loss.item() / ground_truth) mean_loss, std_loss = np.mean(adjusted_losses), np.std(adjusted_losses) # re-weight losses total_loss = 0 for i, loss in enumerate(losses): if adjusted_losses[i] > mean_loss + 1 * std_loss: schedule_factor = scheduler(epoch, args.epochs) total_loss += schedule_factor * loss else: total_loss += loss return total_loss def reduce_loss(loss, reduction): """Reduce loss as specified. Args: loss (Tensor): Elementwise loss tensor. reduction (str): Options are "none", "mean" and "sum". Return: Tensor: Reduced loss tensor. """ # none: 0, elementwise_mean:1, sum: 2 if reduction == 'mean': return loss.mean() elif reduction == 'sum': return loss.sum() else: return loss def weight_reduce_loss(loss, weight=None, reduction='mean', avg_factor=None): """Apply element-wise weight and reduce loss. Args: loss (Tensor): Element-wise loss. weight (Tensor): Element-wise weights. reduction (str): Same as built-in losses of PyTorch. avg_factor (float): Avarage factor when computing the mean of losses. Returns: Tensor: Processed loss values. """ # if weight is specified, apply element-wise weight if weight is not None: loss = loss * weight # if avg_factor is not specified, just reduce the loss if avg_factor is None: loss = reduce_loss(loss, reduction) else: # if reduction is mean, then average the loss by avg_factor if reduction == 'mean': loss = loss.sum() / avg_factor # if reduction is 'none', then do nothing, otherwise raise an error elif reduction != 'none': raise ValueError('avg_factor can not be used with reduction="sum"') return loss def _squeeze_binary_labels(label): if label.size(1) == 1: squeeze_label = label.view(len(label), -1) else: inds = torch.nonzero(label >= 1).squeeze() squeeze_label = inds[:,-1] return squeeze_label def cross_entropy(pred, label, weight=None, reduction='mean', avg_factor=None): # element-wise losses if label.size(-1) != pred.size(0): label = _squeeze_binary_labels(label) loss = F.cross_entropy(pred, label, reduction='none') # apply weights and do the reduction if weight is not None: weight = weight.float() loss = weight_reduce_loss( loss, weight=weight, reduction=reduction, avg_factor=avg_factor) return loss def _expand_binary_labels(labels, label_weights, label_channels): bin_labels = labels.new_full((labels.size(0), label_channels), 0) inds = torch.nonzero(labels >= 1).squeeze() if inds.numel() > 0: bin_labels[inds, labels[inds] - 1] = 1 if label_weights is None: bin_label_weights = None else: bin_label_weights = label_weights.view(-1, 1).expand( label_weights.size(0), label_channels) return bin_labels, bin_label_weights def binary_cross_entropy(pred, label, weight=None, reduction='mean', avg_factor=None): if pred.dim() != label.dim(): label, weight = _expand_binary_labels(label, weight, pred.size(-1)) # weighted element-wise losses if weight is not None: weight = weight.float() loss = F.binary_cross_entropy_with_logits( pred, label.float(), weight, reduction='none') loss = weight_reduce_loss(loss, reduction=reduction, avg_factor=avg_factor) return loss def partial_cross_entropy(pred, label, weight=None, reduction='mean', avg_factor=None): if pred.dim() != label.dim(): label, weight = _expand_binary_labels(label, weight, pred.size(-1)) # weighted element-wise losses if weight is not None: weight = weight.float() mask = label == -1 loss = F.binary_cross_entropy_with_logits( pred, label.float(), weight, reduction='none') if mask.sum() > 0: loss *= (1-mask).float() avg_factor = (1-mask).float().sum() # do the reduction for the weighted loss loss = weight_reduce_loss(loss, reduction=reduction, avg_factor=avg_factor) return loss class ResampleLoss(nn.Module): def __init__(self, class_freq: torch.Tensor, neg_class_freq: torch.Tensor, use_sigmoid=True, reduction='mean', loss_weight=1.0, partial=False, focal=dict( focal=True, balance_param=2.0, gamma=2, ), CB_loss=dict( CB_beta=0.9, CB_mode='average_w' # 'by_class', 'average_n', 'average_w', 'min_n' ), map_param=dict( alpha=10.0, beta=0.2, gamma=0.1 ), logit_reg=dict( neg_scale=5.0, init_bias=0.1 ), reweight_func=None, # None, 'inv', 'sqrt_inv', 'rebalance', 'CB' weight_norm=None # None, 'by_instance', 'by_batch' ): super(ResampleLoss, self).__init__() assert (use_sigmoid is True) or (partial is False) assert class_freq.device == neg_class_freq.device self.use_sigmoid = use_sigmoid self.partial = partial self.loss_weight = loss_weight self.reduction = reduction self.device = class_freq.device if self.use_sigmoid: if self.partial: self.cls_criterion = partial_cross_entropy else: self.cls_criterion = binary_cross_entropy else: self.cls_criterion = cross_entropy # reweighting function self.reweight_func = reweight_func # normalization (optional) self.weight_norm = weight_norm # focal loss params self.focal = focal['focal'] self.gamma = focal['gamma'] self.balance_param = focal['balance_param'] # mapping function params self.map_alpha = map_param['alpha'] self.map_beta = map_param['beta'] self.map_gamma = map_param['gamma'] # CB loss params (optional) self.CB_beta = CB_loss['CB_beta'] self.CB_mode = CB_loss['CB_mode'] self.class_freq = class_freq.float() self.neg_class_freq = neg_class_freq.float() self.num_classes = self.class_freq.shape[0] self.train_num = self.class_freq[0] + self.neg_class_freq[0] # regularization params self.logit_reg = logit_reg self.neg_scale = logit_reg[ 'neg_scale'] if 'neg_scale' in logit_reg else 1.0 init_bias = logit_reg['init_bias'] if 'init_bias' in logit_reg else 0.0 self.init_bias = - torch.log( self.train_num / self.class_freq - 1) * init_bias / self.neg_scale self.freq_inv = torch.ones(self.class_freq.shape).to(self.device) / self.class_freq self.propotion_inv = self.train_num / self.class_freq # print('\033[1;35m loading from {} | {} | {} | s\033[0;0m'.format(freq_file, reweight_func, logit_reg)) # print('\033[1;35m rebalance reweighting mapping params: {:.2f} | {:.2f} | {:.2f} \033[0;0m'.format(self.map_alpha, self.map_beta, self.map_gamma)) def forward(self, cls_score, label, avg_factor=None, **kwargs): weight = self.reweight_functions(label) cls_score, weight = self.logit_reg_functions(label.float(), cls_score, weight) if self.focal: logpt = - self.cls_criterion( cls_score.clone(), label, weight=None, reduction='none', avg_factor=avg_factor) # pt is sigmoid(logit) for pos or sigmoid(-logit) for neg pt = torch.exp(logpt) loss = self.cls_criterion( cls_score, label.float(), weight=weight, reduction='none') loss = ((1 - pt) ** self.gamma) * loss loss = self.balance_param * loss loss = reduce_loss(loss, reduction=self.reduction) else: loss = self.cls_criterion(cls_score, label.float(), weight, reduction=self.reduction) loss = self.loss_weight * loss return loss def reweight_functions(self, label): if self.reweight_func is None: return None elif self.reweight_func in ['inv', 'sqrt_inv']: weight = self.RW_weight(label.float()) elif self.reweight_func in 'rebalance': weight = self.rebalance_weight(label.float()) elif self.reweight_func in 'CB': weight = self.CB_weight(label.float()) else: return None if self.weight_norm is not None: if 'by_instance' in self.weight_norm: max_by_instance, _ = torch.max(weight, dim=-1, keepdim=True) weight = weight / max_by_instance elif 'by_batch' in self.weight_norm: weight = weight / torch.max(weight) return weight def logit_reg_functions(self, labels, logits, weight=None): if not self.logit_reg: return logits, weight if 'init_bias' in self.logit_reg: logits += self.init_bias if 'neg_scale' in self.logit_reg: logits = logits * (1 - labels) * self.neg_scale + logits * labels weight = weight / self.neg_scale * (1 - labels) + weight * labels return logits, weight def rebalance_weight(self, gt_labels): repeat_rate = torch.sum( gt_labels.float() * self.freq_inv, dim=1, keepdim=True) pos_weight = self.freq_inv.clone().detach().unsqueeze(0) / repeat_rate # pos and neg are equally treated weight = torch.sigmoid(self.map_beta * (pos_weight - self.map_gamma)) + self.map_alpha return weight def CB_weight(self, gt_labels): if 'by_class' in self.CB_mode: weight = torch.tensor((1 - self.CB_beta)).to(self.device) / \ (1 - torch.pow(self.CB_beta, self.class_freq)).to(self.device) elif 'average_n' in self.CB_mode: avg_n = torch.sum(gt_labels * self.class_freq, dim=1, keepdim=True) / \ torch.sum(gt_labels, dim=1, keepdim=True) weight = torch.tensor((1 - self.CB_beta)).to(self.device) / \ (1 - torch.pow(self.CB_beta, avg_n)).to(self.device) elif 'average_w' in self.CB_mode: weight_ = torch.tensor((1 - self.CB_beta)).to(self.device) / \ (1 - torch.pow(self.CB_beta, self.class_freq)).to(self.device) weight = torch.sum(gt_labels * weight_, dim=1, keepdim=True) / \ torch.sum(gt_labels, dim=1, keepdim=True) elif 'min_n' in self.CB_mode: min_n, _ = torch.min(gt_labels * self.class_freq + (1 - gt_labels) * 100000, dim=1, keepdim=True) weight = torch.tensor((1 - self.CB_beta)).to(self.device) / \ (1 - torch.pow(self.CB_beta, min_n)).to(self.device) else: raise NameError return weight def RW_weight(self, gt_labels, by_class=True): if 'sqrt' in self.reweight_func: weight = torch.sqrt(self.propotion_inv) else: weight = self.propotion_inv if not by_class: sum_ = torch.sum(weight * gt_labels, dim=1, keepdim=True) weight = sum_ / torch.sum(gt_labels, dim=1, keepdim=True) return weight