File size: 3,823 Bytes
5611f26 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 | import torch
from torch import nn
import torch.nn.functional as F
from math import cos, pi, sin
import math
import numpy as np
from scipy.special import lambertw
def mixup_criterion(criterion, pred, y_a, y_b, lam, pow=2):
y = lam ** pow * y_a + (1 - lam) ** pow * y_b
return criterion(pred, y)
def mixup_data(v, q, a):
'''Returns mixed inputs, pairs of targets, and lambda without organ constraint'''
lam = np.random.beta(1, 1)
batch_size = v.shape[0]
index = torch.randperm(batch_size)
mixed_v = lam * v + (1 - lam) * v[index, :]
mixed_q = lam * q + (1 - lam) * q[index, :]
a_1, a_2 = a, a[index]
return mixed_v, mixed_q, a_1, a_2, lam
def linear(epoch, nepoch):
return 1 - epoch / nepoch
def convex(epoch, nepoch):
return epoch / (2 - nepoch)
def concave(epoch, nepoch):
return 1 - sin((epoch / nepoch) * (pi / 2))
def composite(epoch, nepoch):
return 0.5 * cos((epoch / nepoch) * pi) + 0.5
class LogCoshLoss(nn.Module):
def __init__(self):
super().__init__()
def forward(self, y_t, y_prime_t):
ey_t = y_t - y_prime_t
return torch.mean(torch.log(torch.cosh(ey_t + 1e-12)))+F.mse_loss(y_t, y_prime_t)
class MLCE(nn.Module):
def __init__(self):
super(MLCE, self).__init__()
def _mlcce(self, y_pred, y_true):
y_pred = (1 - 2 * y_true) * y_pred
y_pred_neg = y_pred - y_true * 1e12
y_pred_pos = y_pred - (1 - y_true) * 1e12
zeros = torch.zeros_like(y_pred[..., :1])
y_pred_neg = torch.cat([y_pred_neg, zeros], dim=-1)
y_pred_pos = torch.cat([y_pred_pos, zeros], dim=-1)
neg_loss = torch.logsumexp(y_pred_neg, dim=-1)
pos_loss = torch.logsumexp(y_pred_pos, dim=-1)
loss = torch.mean(neg_loss + pos_loss)
return loss
def __call__(self, y_pred, y_true):
return self._mlcce(y_pred, y_true)
class SuperLoss(nn.Module):
def __init__(self, C=10, lam=1, batch_size=128):
super(SuperLoss, self).__init__()
self.tau = math.log(C)
self.lam = lam # set to 1 for CIFAR10 and 0.25 for CIFAR100
self.batch_size = batch_size
def forward(self, logits, targets):
l_i = F.mse_loss(logits, targets, reduction='none').detach()
sigma = self.sigma(l_i)
loss = (F.mse_loss(logits, targets, reduction='none') - self.tau) * sigma + self.lam * (
torch.log(sigma) ** 2)
loss = loss.sum() / self.batch_size
return loss
def sigma(self, l_i):
x = torch.ones(l_i.size()) * (-2 / math.exp(1.))
x = x.cuda()
y = 0.5 * torch.max(x, (l_i - self.tau) / self.lam)
y = y.cpu().numpy()
sigma = np.exp(-lambertw(y))
sigma = sigma.real.astype(np.float32)
sigma = torch.from_numpy(sigma).cuda()
return sigma
def unbiased_curriculum_loss(out, data, args, epoch, epochs, scheduler='linear'):
losses = []
scheduler = linear if scheduler == 'linear' else concave
# calculate difficulty measurement function
adjusted_losses = []
for idx in range(out.shape[0]):
ground_truth = max(1, abs(data[idx].item()))
loss = F.mse_loss(out[idx], data[idx])
losses.append(loss)
adjusted_losses.append(loss.item() / ground_truth)
mean_loss, std_loss = np.mean(adjusted_losses), np.std(adjusted_losses)
# re-weight losses
total_loss = 0
for i, loss in enumerate(losses):
if adjusted_losses[i] > mean_loss + 1 * std_loss:
schedule_factor = scheduler(epoch, args.epochs)
total_loss += schedule_factor * loss
else:
total_loss += loss
return total_loss
|