Upload 39 files

.gitattributes

@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 MastersThesis_475703.pdf filter=lfs diff=lfs merge=lfs -text
+vlm_eval/__pycache__/run_evaluation.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text

vlm_eval/__pycache__/run_evaluation.cpython-311.pyc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6122c088ad6b90b0847802d1d0eaaefe8b2503bfa8c3c29a370d6c4406b59718
+size 113082

vlm_eval/attacks/apgd.py

@@ -0,0 +1,384 @@
+# Code adapted from https://github.com/chs20/RobustVLM/tree/main
+
+import torch
+import math
+
+
+class APGD:
+    def __init__(self, model, norm, eps, mask_out='context', initial_stepsize=None, decrease_every=None, decrease_every_max=None, random_init=False):
+        # model returns loss sum over batch
+        # thus currently only works with batch size 1
+        # initial_stepsize: in terms of eps. called alpha in apgd
+        # decrease_every: potentially decrease stepsize every x fraction of total iterations. default: 0.22
+        self.model = model
+        self.norm = norm
+        self.eps = eps
+        self.initial_stepsize = initial_stepsize
+        self.decrease_every = decrease_every
+        self.decrease_every_max = decrease_every_max
+        self.random_init = random_init
+        if mask_out != 'none':
+            self.mask_out = mask_out
+        else:
+            self.mask_out = None
+
+    def perturb(self, data_clean, iterations, pert_init=None, verbose=False):
+        mask = self._set_mask(data_clean)
+        data_adv, _, _ = apgd(
+            self.model, data_clean, norm=self.norm, eps=self.eps, n_iter=iterations,
+            use_rs=self.random_init, mask=mask, alpha=self.initial_stepsize,
+            n_iter_2=self.decrease_every, n_iter_min=self.decrease_every_max, pert_init=pert_init,
+            verbose=verbose
+            )
+
+        return data_adv
+
+    def _set_mask(self, data):
+        mask = torch.ones_like(data)
+        if self.mask_out == 'context':
+            mask[:, :-1, ...] = 0
+        elif self.mask_out == 'query':
+            mask[:, -1, ...] = 0
+        elif isinstance(self.mask_out, int):
+            mask[:, self.mask_out, ...] = 0
+        elif self.mask_out is None:
+            pass
+        else:
+            raise NotImplementedError(f'Unknown mask_out: {self.mask_out}')
+        return mask
+
+    def __str__(self):
+        return 'APGD'
+
+
+def L1_projection(x2, y2, eps1):
+    '''
+    x2: center of the L1 ball (bs x input_dim)
+    y2: current perturbation (x2 + y2 is the point to be projected)
+    eps1: radius of the L1 ball
+
+    output: delta s.th. ||y2 + delta||_1 = eps1
+    and 0 <= x2 + y2 + delta <= 1
+    '''
+
+    x = x2.clone().float().view(x2.shape[0], -1)
+    y = y2.clone().float().view(y2.shape[0], -1)
+    sigma = y.clone().sign()
+    u = torch.min(1 - x - y, x + y)
+    # u = torch.min(u, epsinf - torch.clone(y).abs())
+    u = torch.min(torch.zeros_like(y), u)
+    l = -torch.clone(y).abs()
+    d = u.clone()
+
+    bs, indbs = torch.sort(-torch.cat((u, l), 1), dim=1)
+    bs2 = torch.cat((bs[:, 1:], torch.zeros(bs.shape[0], 1).to(bs.device)), 1)
+
+    inu = 2 * (indbs < u.shape[1]).float() - 1
+    size1 = inu.cumsum(dim=1)
+
+    s1 = -u.sum(dim=1)
+
+    c = eps1 - y.clone().abs().sum(dim=1)
+    c5 = s1 + c < 0
+    c2 = c5.nonzero().squeeze(1)
+
+    s = s1.unsqueeze(-1) + torch.cumsum((bs2 - bs) * size1, dim=1)
+    # print(s[0])
+
+    # print(c5.shape, c2)
+
+    if c2.nelement != 0:
+
+        lb = torch.zeros_like(c2).float()
+        ub = torch.ones_like(lb) * (bs.shape[1] - 1)
+
+        # print(c2.shape, lb.shape)
+
+        nitermax = torch.ceil(torch.log2(torch.tensor(bs.shape[1]).float()))
+        counter2 = torch.zeros_like(lb).long()
+        counter = 0
+
+        while counter < nitermax:
+            counter4 = torch.floor((lb + ub) / 2.)
+            counter2 = counter4.type(torch.LongTensor)
+
+            c8 = s[c2, counter2] + c[c2] < 0
+            ind3 = c8.nonzero().squeeze(1)
+            ind32 = (~c8).nonzero().squeeze(1)
+            # print(ind3.shape)
+            if ind3.nelement != 0:
+                lb[ind3] = counter4[ind3]
+            if ind32.nelement != 0:
+                ub[ind32] = counter4[ind32]
+
+            # print(lb, ub)
+            counter += 1
+
+        lb2 = lb.long()
+        alpha = (-s[c2, lb2] - c[c2]) / size1[c2, lb2 + 1] + bs2[c2, lb2]
+        d[c2] = -torch.min(torch.max(-u[c2], alpha.unsqueeze(-1)), -l[c2])
+
+    return (sigma * d).view(x2.shape)
+
+def L0_projection(x_adv, x, eps, step_size, lam=0.01):
+
+    
+    pert = x_adv - x
+    
+    pert_proj = torch.clamp(pert,-eps,eps)
+    x_adv_temp = torch.clamp(x + pert_proj,0.,1.)
+    pert_proj = x_adv_temp - x
+    pert = torch.where(pert ** 2 - (pert_proj - pert) ** 2 > 2 * step_size * lam, pert_proj, 0)
+    #pert = torch.where(pert > (2 * lam * step_size) ** 0.5, pert, 0)
+    return torch.clamp(x+pert,0.0,1.0)
+
+
+
+def L1_norm(x, keepdim=False):
+    z = x.abs().view(x.shape[0], -1).sum(-1)
+    if keepdim:
+        z = z.view(-1, *[1] * (len(x.shape) - 1))
+    return z
+
+
+def L2_norm(x, keepdim=False):
+    z = (x ** 2).view(x.shape[0], -1).sum(-1).sqrt()
+    if keepdim:
+        z = z.view(-1, *[1] * (len(x.shape) - 1))
+    return z
+
+
+def L0_norm(x):
+    return (x != 0.).view(x.shape[0], -1).sum(-1)
+
+
+def dlr_loss(x, y, reduction='none'):
+    x_sorted, ind_sorted = x.sort(dim=1)
+    ind = (ind_sorted[:, -1] == y).float()
+
+    return -(x[torch.arange(x.shape[0]), y] - x_sorted[:, -2] * ind - \
+             x_sorted[:, -1] * (1. - ind)) / (x_sorted[:, -1] - x_sorted[:, -3] + 1e-12)
+
+
+def dlr_loss_targeted(x, y, y_target):
+    x_sorted, ind_sorted = x.sort(dim=1)
+    u = torch.arange(x.shape[0])
+
+    return -(x[u, y] - x[u, y_target]) / (x_sorted[:, -1] - .5 * (
+            x_sorted[:, -3] + x_sorted[:, -4]) + 1e-12)
+
+def check_oscillation(x, j, k, y5, k3=0.75):
+    t = torch.zeros(x.shape[1]).to(x.device)
+    for counter5 in range(k):
+        t += (x[j - counter5] > x[j - counter5 - 1]).float()
+
+    return (t <= k * k3 * torch.ones_like(t)).float()
+
+
+def apgd(model, x, norm, eps, n_iter=10, use_rs=False, mask=None, alpha=None, n_iter_2=None,
+               n_iter_min=None, pert_init=None, verbose=False, is_train=True):
+    # from https://github.com/fra31/robust-finetuning
+    assert x.shape[0] == 1  # only support batch size 1 for now
+    norm = norm.replace('l', 'L')
+    device = x.device
+    ndims = len(x.shape) - 1
+
+    if not use_rs:
+        x_adv = x.clone()
+    else:
+        if norm == 'Linf':
+            t = torch.zeros_like(x).uniform_(-eps, eps).detach()
+            x_adv = x + t
+        elif norm == 'L2':
+            t = torch.randn(x.shape).to(device).detach()
+            x_adv = x + eps * torch.ones_like(x).detach() * t / (L2_norm(t, keepdim=True) + 1e-12)
+    if pert_init is not None:
+        assert not use_rs
+        assert pert_init.shape == x.shape, f'pert_init.shape: {pert_init.shape}, x.shape: {x.shape}'
+        x_adv = x + pert_init
+
+    x_adv = x_adv.clamp(0., 1.)
+    x_best = x_adv.clone()
+    x_best_adv = x_adv.clone()
+    loss_steps = torch.zeros([n_iter, x.shape[0]], device=device)
+    loss_best_steps = torch.zeros([n_iter + 1, x.shape[0]], device=device)
+
+    # set params
+    n_fts = math.prod(x.shape[1:])
+    if norm in ['Linf', 'L2']:
+        n_iter_2_frac = 0.22 if n_iter_2 is None else n_iter_2
+        n_iter_min_frac = 0.06 if n_iter_min is None else n_iter_min
+        n_iter_2 = max(int(n_iter_2_frac * n_iter), 1)
+        n_iter_min = max(int(n_iter_min_frac * n_iter), 1)
+        size_decr = max(int(0.03 * n_iter), 1)
+        k = n_iter_2 + 0
+        thr_decr = .75
+        alpha = 2. if alpha is None else alpha
+    elif norm in ['L1','L0']:
+        k = max(int(.04 * n_iter), 1)
+        init_topk = .05 if is_train else .2
+        topk = init_topk * torch.ones([x.shape[0]], device=device)
+        sp_old = n_fts * torch.ones_like(topk)
+        adasp_redstep = 1.5
+        adasp_minstep = 10.
+        alpha = 1. if alpha is None else alpha
+
+    step_size = alpha * eps * torch.ones([x.shape[0], *[1] * ndims],
+                                         device=device)
+    counter3 = 0
+
+    x_adv.requires_grad_()
+    # grad = torch.zeros_like(x)
+    # for _ in range(self.eot_iter)
+    with torch.enable_grad():
+        loss_indiv = model(x_adv)#.unsqueeze(0)
+        loss = loss_indiv.sum()
+    # grad += torch.autograd.grad(loss, [x_adv])[0].detach()
+    grad = torch.autograd.grad(loss, [x_adv])[0].detach()
+    if mask is not None:
+        grad *= mask
+    # grad /= float(self.eot_iter)
+    grad_best = grad.clone()
+    x_adv.detach_()
+    loss_indiv = loss_indiv.detach()
+    loss = loss.detach()
+
+    loss_best = loss_indiv.detach().clone()
+    loss_best_last_check = loss_best.clone()
+    reduced_last_check = torch.ones_like(loss_best)
+    n_reduced = 0
+
+    u = torch.arange(x.shape[0], device=device)
+    x_adv_old = x_adv.clone().detach()
+
+    for i in range(n_iter):
+        ### gradient step
+        if True:  # with torch.no_grad()
+            x_adv = x_adv.detach()
+            grad2 = x_adv - x_adv_old
+            x_adv_old = x_adv.clone()
+            loss_curr = loss.detach().mean()
+
+            a = 0.75 if i > 0 else 1.0
+
+            if norm == 'Linf':
+                x_adv_1 = x_adv + step_size * torch.sign(grad)
+                x_adv_1 = torch.clamp(torch.min(torch.max(x_adv_1,
+                                                          x - eps), x + eps), 0.0, 1.0)
+                x_adv_1 = torch.clamp(torch.min(torch.max(
+                    x_adv + (x_adv_1 - x_adv) * a + grad2 * (1 - a),
+                    x - eps), x + eps), 0.0, 1.0)
+
+            elif norm == 'L2':
+                x_adv_1 = x_adv + step_size * grad / (L2_norm(grad,
+                                                              keepdim=True) + 1e-12)
+                x_adv_1 = torch.clamp(x + (x_adv_1 - x) / (L2_norm(x_adv_1 - x,
+                                                                   keepdim=True) + 1e-12) * torch.min(eps * torch.ones_like(x),
+                                                                                                      L2_norm(x_adv_1 - x, keepdim=True)), 0.0, 1.0)
+                x_adv_1 = x_adv + (x_adv_1 - x_adv) * a + grad2 * (1 - a)
+                x_adv_1 = torch.clamp(x + (x_adv_1 - x) / (L2_norm(x_adv_1 - x,
+                                                                   keepdim=True) + 1e-12) * torch.min(eps * torch.ones_like(x),
+                                                                                                      L2_norm(x_adv_1 - x, keepdim=True)), 0.0, 1.0)
+
+            elif norm == 'L1':
+                grad_topk = grad.abs().view(x.shape[0], -1).sort(-1)[0]
+                topk_curr = torch.clamp((1. - topk) * n_fts, min=0, max=n_fts - 1).long()
+                grad_topk = grad_topk[u, topk_curr].view(-1, *[1] * (len(x.shape) - 1))
+                sparsegrad = grad * (grad.abs() >= grad_topk).float()
+                x_adv_1 = x_adv + step_size * sparsegrad.sign() / (
+                        sparsegrad.sign().abs().view(x.shape[0], -1).sum(dim=-1).view(
+                            -1, 1, 1, 1) + 1e-10)
+
+                delta_u = x_adv_1 - x
+                delta_p = L1_projection(x, delta_u, eps)
+                x_adv_1 = x + delta_u + delta_p
+
+            elif norm == 'L0':
+                L1normgrad = grad / (grad.abs().view(grad.shape[0], -1).sum(
+                    dim=-1, keepdim=True) + 1e-12).view(grad.shape[0], *[1] * (
+                        len(grad.shape) - 1))
+                x_adv_1 = x_adv + step_size * L1normgrad * n_fts
+                # TODO: add momentum
+
+            x_adv = x_adv_1.to(dtype=x_adv.dtype) + 0.
+
+        ### get gradient
+        x_adv.requires_grad_()
+        # grad = torch.zeros_like(x)
+        # for _ in range(self.eot_iter)
+        with torch.enable_grad():
+            loss_indiv = model(x_adv)#.unsqueeze(0)
+            loss = loss_indiv.sum()
+
+        # grad += torch.autograd.grad(loss, [x_adv])[0].detach()
+        if i < n_iter - 1:
+            # save one backward pass
+            grad = torch.autograd.grad(loss, [x_adv])[0].detach()
+            if mask is not None:
+                grad *= mask
+        # grad /= float(self.eot_iter)
+        x_adv.detach_()
+        loss_indiv = loss_indiv.detach()
+        loss = loss.detach()
+
+        x_best_adv = x_adv + 0.
+        if verbose and (i % max(n_iter // 10, 1) == 0 or i == n_iter - 1):
+            str_stats = ' - step size: {:.5f} - topk: {:.2f}'.format(
+                step_size.mean(), topk.mean() * n_fts) if norm in ['L1'] else ' - step size: {:.5f}'.format(
+                step_size.mean())
+            print('iteration: {} - best loss: {:.6f} curr loss {:.6f} {}'.format(
+                i, loss_best.sum(), loss_curr, str_stats))
+            # print('pert {}'.format((x - x_best_adv).abs().view(x.shape[0], -1).sum(-1).max()))
+
+        ### check step size
+        if True:  # with torch.no_grad()
+            y1 = loss_indiv.detach().clone()
+            loss_steps[i] = y1 + 0
+            ind = (y1 > loss_best).nonzero().squeeze()
+            x_best[ind] = x_adv[ind].clone()
+            grad_best[ind] = grad[ind].clone()
+            loss_best[ind] = y1[ind] + 0
+            loss_best_steps[i + 1] = loss_best + 0
+
+            counter3 += 1
+
+            if counter3 == k:
+                if norm in ['Linf', 'L2']:
+                    fl_oscillation = check_oscillation(loss_steps, i, k,
+                                                       loss_best, k3=thr_decr)
+                    fl_reduce_no_impr = (1. - reduced_last_check) * (
+                            loss_best_last_check >= loss_best).float()
+                    fl_oscillation = torch.max(fl_oscillation,
+                                               fl_reduce_no_impr)
+                    reduced_last_check = fl_oscillation.clone()
+                    loss_best_last_check = loss_best.clone()
+
+                    if fl_oscillation.sum() > 0:
+                        ind_fl_osc = (fl_oscillation > 0).nonzero().squeeze()
+                        step_size[ind_fl_osc] /= 2.0
+                        n_reduced = fl_oscillation.sum()
+
+                        x_adv[ind_fl_osc] = x_best[ind_fl_osc].clone()
+                        grad[ind_fl_osc] = grad_best[ind_fl_osc].clone()
+
+                    counter3 = 0
+                    k = max(k - size_decr, n_iter_min)
+
+                elif norm in ['L1']:
+                    # adjust sparsity
+                    sp_curr = L0_norm(x_best - x)
+                    fl_redtopk = (sp_curr / sp_old) < .95
+                    topk = sp_curr / n_fts / 1.5
+                    step_size[fl_redtopk] = alpha * eps
+                    step_size[~fl_redtopk] /= adasp_redstep
+                    step_size.clamp_(alpha * eps / adasp_minstep, alpha * eps)
+                    sp_old = sp_curr.clone()
+
+                    x_adv[fl_redtopk] = x_best[fl_redtopk].clone()
+                    grad[fl_redtopk] = grad_best[fl_redtopk].clone()
+
+                    counter3 = 0
+    
+    return x_best, loss_best, x_best_adv
+
+

vlm_eval/attacks/attack.py

@@ -0,0 +1,20 @@
+import torch
+
+
+class Attack(object):
+    '''
+    Root class for all adversarial attack classes.
+    '''
+
+    def __init__(self, model, targeted=False, img_range=(0, 1)):
+        self.model = model
+        self.device = 'cuda:0'
+        self.targeted = targeted
+        self.img_range = img_range
+
+    def __repr__(self):
+        return str(self.__dict__)
+
+    def to(self, device):
+        self.model.to(device)
+        self.device = device

vlm_eval/attacks/ead.py

@@ -0,0 +1,132 @@
+# Code taken and adapted from https://github.com/wagnermoritz/GSE
+import torch
+from vlm_eval.attacks.attack import Attack
+
+class EAD(Attack):
+
+    def __init__(self,model, targeted=False, img_range=(0,1), steps=100, beta=5e-5, mask_out='none', ver=False, binary_steps=2, step_size=1e-2, decision_rule='L1'):
+        
+        super().__init__(model=model, targeted=targeted, img_range=img_range)
+        self.steps = steps
+        self.ver = ver
+        self.binary_steps = binary_steps
+        self.beta = beta
+        if mask_out != 'none':
+            self.mask_out = mask_out
+        else:
+            self.mask_out = None
+        self.decision_rule = decision_rule
+        self.ver = ver
+        self.step_size = step_size
+
+    def _set_mask(self, data):
+        mask = torch.ones_like(data)
+        if self.mask_out == 'context':
+            mask[:, :-1, ...] = 0
+        elif self.mask_out == 'query':
+            mask[:, -1, ...] = 0
+        elif isinstance(self.mask_out, int):
+            mask[:, self.mask_out, ...] = 0
+        elif self.mask_out is None:
+            pass
+        else:
+            raise NotImplementedError(f'Unknown mask_out: {self.mask_out}')
+        return mask
+
+    def __call__(self, x_orig):
+        
+        for param in self.model.model.parameters():
+            param.requires_grad = False
+
+        mask_out = self._set_mask(x_orig)
+
+        c = 1e-1
+        c_upper = 10e+10
+        c_lower = 0
+        
+        overall_best_attack = x_orig.clone()
+        overall_best_dist = torch.inf
+        overall_best_loss = 1e10
+
+        for binary_step in range(self.binary_steps):
+
+            global_step = 0      
+            x = x_orig.clone().detach()
+            y = x_orig.clone().detach()   
+
+            best_attack = x_orig.clone().detach()
+            best_dist = torch.inf
+            best_loss = 1e10
+
+            step_size = 1e-2
+
+            for step in range(self.steps):
+
+                y.requires_grad = True
+                _, loss = self.loss_fn(x=y, c=c, x_orig=x_orig)
+                loss.backward()
+                y_grad = y.grad.data * mask_out
+
+                with torch.no_grad():
+                    x_new = self.project(x=y-step_size*y_grad, x_orig=x_orig)
+
+                    step_size = (self.step_size - 0) * (1 - global_step / self.steps) ** 0.5 + 0
+                    global_step += 1
+                    
+                    y = x_new + (step / (step + 3)) * (x_new - x)
+                    x = x_new
+                                    
+                    loss_model, loss = self.loss_fn(x=x, c=c, x_orig=x_orig)
+
+                    if self.ver and step % 20 == 0:
+                        print(f"Binary Step: {binary_step}, Iter: {step}, Loss: {loss.item()}, L0: {(x - x_orig).norm(p=0)}, Linf: {(x - x_orig).norm(p=torch.inf)}")
+
+                    if self.decision_rule == 'L1':
+                        if (x - x_orig).norm(p=1).item() < best_dist and loss_model < best_loss:
+                            best_loss = loss_model
+                            best_attack = x.clone()
+                            best_dist = (x - x_orig).norm(p=1).item()
+                    else:
+                        raise NotImplementedError
+
+            # Updating c
+            if overall_best_dist > best_dist and best_loss < overall_best_loss:
+                overall_best_loss = best_loss
+                overall_best_dist = best_dist
+                overall_best_attack = best_attack.clone()
+
+                c_upper = min(c_upper, c)
+                if c_upper < 1e9:
+                    c = (c_upper + c_lower) / 2
+
+            else:
+                c_lower = max(c_lower, c)
+                if c_upper < 1e9:
+                    c = (c_lower + c_upper) / 2.0
+                else:
+                    c *= 10
+        
+        print(f"Final L0: {(overall_best_attack - x_orig).norm(p=0)}, Linf: {(overall_best_attack - x_orig).norm(p=torch.inf)}")
+        return overall_best_attack.detach()
+
+
+    def project(self, x, x_orig):
+
+        mask_1 = (x - x_orig > self.beta).float()
+        mask_2 = ((x - x_orig).abs() <= self.beta).float()
+        mask_3 = (x - x_orig < -self.beta).float()
+
+        upper = torch.minimum(x - self.beta, torch.tensor(1.0))
+        lower = torch.maximum(x + self.beta, torch.tensor(0.0))
+        
+        proj_x = mask_1 * upper + mask_2 * x_orig + mask_3 * lower
+        return proj_x   
+    
+    def loss_fn(self, x, c, x_orig):
+        
+        out = -self.model(x).sum() if not self.targeted else self.model(x).sum()
+        l2_dist = ((x - x_orig) ** 2).view(x.shape[0], -1).sum(dim=1)
+        l1_dist = ((x - x_orig).abs()).view(x.shape[0], -1).sum(dim=1)
+
+        return out, c * out + l2_dist.sum() + \
+                    self.beta * l1_dist.sum()

vlm_eval/attacks/fwnucl.py

@@ -0,0 +1,170 @@
+# Code taken and adapted from https://github.com/wagnermoritz/GSE
+import torch
+import math
+from vlm_eval.attacks.attack import Attack
+
+class FWnucl(Attack):
+    def __init__(self, model, *args, iters=200, img_range=(-1, 1), ver=False,
+                 targeted=False, eps=5, mask_out='none',**kwargs):
+        '''
+        Implementation of the nuclear group norm attack.
+
+        args:
+        model:         Callable, PyTorch classifier.
+        ver:           Bool, print progress if True.
+        img_range:     Tuple of ints/floats, lower and upper bound of image
+                       entries.
+        targeted:      Bool, given label is used as a target label if True.
+        eps:           Float, radius of the nuclear group norm ball.
+        '''
+        super().__init__(model, img_range=img_range, targeted=targeted)
+        self.iters = iters
+        self.ver = ver
+        self.eps = eps
+        self.gr = (math.sqrt(5) + 1) / 2
+        if mask_out != 'none':
+            self.mask_out = mask_out
+        else:
+            self.mask_out = None
+
+    def _set_mask(self, data):
+        mask = torch.ones_like(data)
+        if self.mask_out == 'context':
+            mask[:, :-1, ...] = 0
+        elif self.mask_out == 'query':
+            mask[:, -1, ...] = 0
+        elif isinstance(self.mask_out, int):
+            mask[:, self.mask_out, ...] = 0
+        elif self.mask_out is None:
+            pass
+        else:
+            raise NotImplementedError(f'Unknown mask_out: {self.mask_out}')
+        return mask
+
+
+    def __loss_fn(self, x):
+        '''
+        Compute loss depending on self.targeted.
+        '''
+        if self.targeted:
+            return -self.model(x).sum()
+        else:
+            return self.model(x).sum()
+
+
+    def __call__(self, x, *args, **kwargs):
+        '''
+        Perform the nuclear group norm attack on a batch of images x.
+
+        args:
+        x:   Tensor of shape [B, C, H, W], batch of images.
+        y:   Tensor of shape [B], batch of labels.
+
+        Returns a tensor of the same shape as x containing adversarial examples
+        '''
+
+        for param in self.model.model.parameters():
+            param.requires_grad = False
+
+        mask_out = self._set_mask(x)
+        x = x.to(self.device)
+        noise = torch.zeros_like(x)
+        noise.requires_grad = True
+
+        for t in range(self.iters):
+            if self.ver:
+                print(f'\rIteration {t+1}/{self.iters}', end='')
+            
+            loss = self.__loss_fn(x + noise * mask_out)
+            loss.backward()
+            noise.grad.data = noise.grad.data * mask_out
+            s = self.__groupNuclearLMO(noise.grad.data, eps=self.eps)
+            with torch.no_grad():
+                gamma = self.__lineSearch(x=x, s=s, noise=noise)
+                noise = (1 - gamma) * noise + gamma * s
+            noise.requires_grad = True
+
+            if self.ver and t % 20 == 0:
+                print(f"Iteration: {t}, Loss: {loss.item()}")
+        x = torch.clamp(x + noise, 0, 1)
+        if self.ver:
+            print("")
+        return x.detach()
+
+
+    def __lineSearch(self, x, s, noise, steps=25):
+        '''
+        Perform line search for the step size.
+        '''
+        a = torch.zeros(x.shape[1], device=self.device).view(-1, 1, 1, 1)
+        b = torch.ones(x.shape[1], device=self.device).view(-1, 1, 1, 1)
+        c = b - (b - a) / self.gr
+        d = a + (b - a) / self.gr
+        sx = s - noise
+
+        for i in range(steps):
+            loss1 = self.__loss_fn(x + noise + (c * sx).view(*x.shape))
+            loss2 = self.__loss_fn(x + noise + (d * sx).view(*x.shape))
+            mask = loss1 > loss2
+
+            b[mask] = d[mask]
+            mask = torch.logical_not(mask)
+            a[mask] = c[mask]
+
+            c = b - (b - a) / self.gr
+            d = a + (b - a) / self.gr
+
+        return (b + a) / 2
+
+
+    def __groupNuclearLMO(self, x, eps=5):
+        '''
+        LMO for the nuclear group norm ball.
+        '''
+
+        B, C, H, W = x.shape[1], x.shape[3], x.shape[4], x.shape[5]
+        size = 32 if H > 64 else 4
+
+        # turn batch of images into batch of size by size pixel groups per
+        # color channel
+        xrgb = [x.view(B, C, H, W)[:, c, :, :] for c in range(C)]
+        xrgb = [xc.unfold(1, size, size).unfold(2, size, size) for xc in xrgb]
+        xrgb = [xc.reshape(-1, size, size) for xc in xrgb]
+
+        # compute nuclear norm of each patch (sum norms over color channels)
+        norms = torch.linalg.svdvals(xrgb[0])
+        for xc in xrgb[1:]:
+            norms += torch.linalg.svdvals(xc)
+        norms = norms.sum(-1).reshape(B, -1)
+
+        # only keep the patch g* with the largest nuclear norm for each image
+        idxs = norms.argmax(dim=1).view(-1, 1)
+        xrgb = [xc.reshape(B, -1, size, size) for xc in xrgb]
+        xrgb = [xc[torch.arange(B).view(-1, 1), idxs].view(B, size, size)
+                for xc in xrgb]
+
+        # build index tensor corr. to the position of the kept patches in x
+        off = (idxs % (W / size)).long() * size
+        off += torch.floor(idxs / (W / size)).long() * W * size
+        idxs = torch.arange(0, size**2,
+                            device=self.device).view(1, -1).repeat(B, 1) + off
+        off = torch.arange(0, size,
+                           device=self.device).view(-1, 1).repeat(1, size)
+        off = off * W  - off * size
+        idxs += off.view(1, -1)
+
+        # compute singular vector pairs corresponding to largest singular value
+        # and final perturbation (LMO solution)
+        pert = torch.zeros_like(x).view(B, C, H, W)
+        for i, xc in enumerate(xrgb):
+            U, _, V = torch.linalg.svd(xc)
+            U = U[:, :, 0].view(B, size, 1)
+            V = V.transpose(-2, -1)[:, :, 0].view(B, size, 1)
+            pert_gr = torch.bmm(U, V.transpose(-2, -1)).reshape(B, size * size)
+            idx = torch.arange(B).view(-1, 1)
+            pert_tmp = pert[:, i, :, :].view(B, -1)
+            pert_tmp[idx, idxs] = pert_gr * eps
+            pert_clone = pert.clone()
+            pert_clone[:, i, :, :] = pert_tmp.view(B, H, W)
+
+        return pert_clone.view(*x.shape)

vlm_eval/attacks/gse.py

@@ -0,0 +1,313 @@
+# Code taken and adapted from https://github.com/wagnermoritz/GSE
+import torch
+import torchvision
+import math
+import torch.nn.functional as F
+
+from vlm_eval.attacks.attack import Attack
+
+
+# required input size : batch_size x num_media x num_frames x channels x height x width 
+class GSEAttack(Attack):
+    def __init__(self, model, *args, mask_out='none',ver=False, img_range=(-1, 1), search_steps=4,
+                 targeted=False, sequential=False, search_factor=2,
+                 gb_size=5, sgm=1.5, mu=1, sigma=0.0025, iters=200, k_hat=10,
+                 q=0.25, **kwargs):
+        '''
+        Implementation of the GSE attack.
+
+        args:
+        model:         Callable, PyTorch classifier.
+        mask_out:      Masks out context images if set to context, query images if set to query and none if set to none.
+        ver:           Bool, print progress if True.
+        img_range:     Tuple of ints/floats, lower and upper bound of image
+                       entries.
+        search_steps:  Int, number of steps for line search on the trade-off
+                       parameter.
+        targeted:      Bool, given label is used as a target label if True.
+        sequential:    Bool, perturbations are computed sequentially for all
+                       images in the batch if True. For fair comparison to
+                       Homotopy attack.
+        search_factor: Float, factor to increase/decrease the trade-off
+                       parameter until an upper/lower bound for the line search
+                       is found.
+        gb_size:       Odd int, size of the Gaussian blur kernel.
+        sgm:           Float, sigma of the gaussian blur kernel
+        mu:            Float, trade-off parameter for 2-norm regularization.
+        sigma:         Float, step size
+        iters:         Int, number of iterations.
+        k_hat:         Int, number of iterations before transitioning to NAG.
+        q:             Float, inverse of increase factor for adjust_lambda.
+        '''
+        super().__init__(model, img_range=img_range, targeted=targeted)
+        self.ver = ver
+        self.search_steps = search_steps
+        self.sequential = sequential
+        self.search_factor = search_factor
+        self.gb_size = gb_size
+        self.sgm = sgm
+        self.mu = mu
+        self.sigma = sigma
+        self.iters = iters
+        self.k_hat = k_hat
+        self.q = q
+        if mask_out != 'none':
+            self.mask_out = mask_out
+        else:
+            self.mask_out = None
+
+    def adjust_lambda(self, lam, noise):
+        '''
+        Adjust trade-off parameters (lambda) to update search space.
+        '''
+        x = noise.detach().clone().abs().mean(dim=1, keepdim=True).sign()
+        gb = torchvision.transforms.GaussianBlur((self.gb_size, self.gb_size),
+                                                 sigma=self.sgm)
+        x = gb(x) + 1
+        x = torch.where(x == 1, self.q, x)
+        lam /= x[:, 0, :, :]
+        return lam
+
+
+    def section_search(self, x, steps=50):
+        '''
+        Section search for finding the maximal lambda such that the
+        perturbation is non-zero after the first iteration.
+        '''
+
+        noise = torch.zeros_like(x, requires_grad=True) # the shape of 'x' is batch_size x num_media x num_frames x Color x height x width
+        loss = (-self.model(x + noise).sum() + self.mu
+                * torch.norm(noise.view(x.size(1), x.size(3), x.size(4), x.size(5)), p=2, dim=(1,2,3)).sum())
+        grad = torch.autograd.grad(loss, [noise])[0].detach()
+        noise.detach_()
+        ones = torch.ones_like(x.view(x.size(1), x.size(3), x.size(4), x.size(5)))[:, 0, :, :]
+
+        # define upper and lower bound for line search
+        lb = torch.zeros((x.size(1),), dtype=torch.float,
+                         device=self.device).view(-1, 1, 1)
+        ub = lb.clone() + 0.001
+        mask = torch.norm(self.prox(grad.clone().view(x.size(1),x.size(3),x.size(4),x.size(5)) * self.sigma,
+                                      ones * ub * self.sigma),
+                          p=0, dim=(1,2,3)) != 0
+        while mask.any():
+            ub[mask] *= 2
+            mask = torch.norm(self.prox(grad.clone().view(x.size(1),x.size(3),x.size(4),x.size(5)) * self.sigma,
+                                          ones * ub * self.sigma),
+                              p=0, dim=(1,2,3)) != 0
+
+        # perform search
+        for _ in range(steps):
+            cur = (ub + lb) / 2
+            mask = torch.norm(self.prox(grad.clone().view(x.size(1),x.size(3),x.size(4),x.size(5)) * self.sigma,
+                                          ones * cur * self.sigma),
+                              p=0, dim=(1,2,3)) == 0
+            ub[mask] = cur[mask]
+            mask = torch.logical_not(mask)
+            lb[mask] = cur[mask]
+        cur = (lb + ub).view(-1) / 2
+        return 0.01 * cur
+
+
+    def __call__(self, x, y, *args, **kwargs):
+        '''
+        Call the attack for a batch of images x or sequentially for all images
+        in x depending on self.sequential.
+
+        args:
+        x:   Tensor of shape [B, C, H, W], batch of images.
+        y:   Tensor of shape [B], batch of labels.
+
+        Returns a tensor of the same shape as x containing adversarial examples
+        '''
+        if self.sequential:
+            result = x.clone()
+            for i, (x_, y_) in enumerate(zip(x, y)):
+                result[i] = self.perform_att(x_.unsqueeze(0),
+                                             y_.unsqueeze(0),
+                                             mu=self.mu, sigma=self.sigma,
+                                             k_hat=self.k_hat).detach()
+            return result
+        else:
+            return self.perform_att(x, y, mu=self.mu, sigma=self.sigma,
+                                    k_hat=self.k_hat)
+
+
+    def _set_mask(self, data):
+        mask = torch.ones_like(data)
+        if self.mask_out == 'context':
+            mask[:, :-1, ...] = 0
+        elif self.mask_out == 'query':
+            mask[:, -1, ...] = 0
+        elif isinstance(self.mask_out, int):
+            mask[:, self.mask_out, ...] = 0
+        elif self.mask_out is None:
+            pass
+        else:
+            raise NotImplementedError(f'Unknown mask_out: {self.mask_out}')
+        return mask
+
+
+    def perform_att(self, x, mu, sigma, k_hat):
+        '''
+        Perform GSE attack on a batch of images x with corresponding labels y.
+        '''
+        x = x.to(self.device)
+        B, C, H, W = x.shape[1], x.shape[3], x.shape[4], x.shape[5] # Input is of the shape Batch x Num_media x num_frames x colors x height x width
+        lams = self.section_search(x)
+        mask_out = self._set_mask(x).view(B,C,H,W)
+        # save x, y, and lams for resetting them at the beginning of every
+        # section search step
+        save_x = x.clone()
+        save_lams = lams.clone()
+        # upper and lower bounds for section learch
+        ub_lams = torch.full_like(lams, torch.inf)
+        lb_lams = torch.full_like(lams, 0.0)
+        # tensor for saving succesful adversarial examples in inner loop
+        result = x.clone()
+        # tensor for saving best adversarial example so far
+        result2 = x.clone()
+        best_l0 = torch.full((B,), torch.inf, device=self.device).type(x.type())
+
+        # section search
+        for step in range(self.search_steps):
+            x = save_x.clone()
+            lams = save_lams.clone()
+            lam = torch.ones_like(x.view(B, C, H, W))[:, 0, :, :] * lams.view(-1, 1, 1)
+            # tensor for tracking for which images adv. examples have been found
+            active = torch.ones(B, dtype=bool, device=self.device)
+            # set initial perturbation to zero
+            noise = torch.zeros_like(x, requires_grad = True)
+            noise_old = noise.clone()
+            lr = 1
+
+            # attack
+            for j in range(self.iters):
+                if self.ver:
+                    print(f'\rSearch step {step + 1}/{self.search_steps}, ' +
+                          f'Prox.Grad. Iteration {j + 1}/{self.iters}, ' +
+                          f'Images left: {x.shape[1]}', end='')
+                if len(x) == 0:
+                    break
+
+                self.model.model.zero_grad()
+                loss = (-self.model(x + noise).sum() + mu
+                        * (torch.norm(noise.view(B, C, H, W), p=2, dim=(1,2,3)) ** 2).sum())
+                noise_grad_data = torch.autograd.grad(loss, [noise])[0].detach().view(B, C, H, W)
+                #print(f"{loss} {(torch.norm(noise.view(B, C, H, W), p=2, dim=(1,2,3)) ** 2).sum()}")
+                with torch.no_grad():
+                    
+                    noise_grad_data = noise_grad_data * mask_out # Mask_out shape B x C x H x W
+                    lr_ = (1 + math.sqrt(1 + 4 * lr**2)) / 2
+                    if j == k_hat:
+                        lammask = (lam > lams.view(-1, 1, 1))[:, None, :, :]
+                        lammask = lammask.repeat(1, C, 1, 1)
+                        noise_old = noise.clone()
+                    if j < k_hat:
+                        noise = noise - sigma * noise_grad_data.view(1, B, 1, C, H, W)
+                        noise = self.prox(noise.view(B, C, H, W), lam * sigma).view(1, B, 1, C, H, W)
+                        noise_tmp = noise.clone()
+                        noise = lr / lr_ * noise + (1 - (lr/ lr_)) * noise_old
+                        noise_old = noise_tmp.clone()
+                        lam = self.adjust_lambda(lam, noise.view(B, C, H, W))
+                    else:
+                        noise = noise - sigma * noise_grad_data.view(1, B, 1, C, H, W)
+                        noise_tmp = noise.clone()
+                        noise = lr / lr_ * noise + (1 - (lr/ lr_)) * noise_old
+                        noise_old = noise_tmp.clone()
+                        noise[lammask.view(1, B, 1, C, H, W)] = 0
+                    # clamp adv. example to valid range
+                    x_adv = torch.clamp(x + noise, *self.img_range)
+                    noise = x_adv - x
+                    lr = lr_
+                    
+
+                noise.requires_grad = True
+            
+            # section search
+            # no adv. example found => decrease upper bound and current lambda
+            # adv. example found => save it if the "0-norm" is better than of the
+            # previous adv. example, increase lower bound and current lambda
+            for i in range(B):
+                if active[i]:
+                    ub_lams[i] = save_lams[i]
+                    save_lams[i] = 0.95 * lb_lams[i] + 0.05 * save_lams[i]
+                else:
+                    print("here")
+                    l0 = self.l20((result[i] - save_x[i]).unsqueeze(0)).to(self.device)
+                    if l0 < best_l0[i]:
+                        best_l0[i] = l0
+                        result2[i] = result[i].clone()
+                    if torch.isinf(ub_lams[i]):
+                        lb_lams[i] = save_lams[i]
+                        save_lams[i] *= self.search_factor
+                    else:
+                        lb_lams[i] = save_lams[i]
+                        save_lams[i] = (ub_lams[i] + save_lams[i]) / 2
+
+        if self.ver:
+            print('')
+        
+        return x_adv
+
+    def extract_patches(self, x):
+        '''
+        Extracts and returns all overlapping size by size patches from
+        the image batch x.
+        '''
+        B, C, _, _ = x.shape
+        size = 8
+        kernel = torch.zeros((size ** 2, size ** 2))
+        kernel[range(size**2), range(size**2)] = 1.0
+        kernel = kernel.view(size**2, 1, size, size)
+        kernel = kernel.repeat(C, 1, 1, 1).to(x.device)
+        out = F.conv2d(x, kernel, groups=C)
+        out = out.view(B, C, size, size, -1)
+        out = out.permute(0, 4, 1, 2, 3)
+        return out.contiguous()
+    
+    def l20(self, x):
+        '''
+        Computes d_{2,0}(x[i]) for all perturbations x[i] in the batch x
+        as described in section 3.2.
+        '''
+        B, N, M, C, _, _ = x.shape
+        l20s = []
+    
+        for b in range(B):
+            for n in range(N):
+                for m in range(M):
+                    x_ = x[b, n, m]  # Select the specific perturbation x[b, n, m]
+                    patches = self.extract_patches(x_.unsqueeze(0))  # Add unsqueeze to match 6D input
+                    l2s = torch.norm(patches, p=2, dim=(2,3,4))
+                    l20s.append((l2s != 0).float().sum().item())
+    
+        return torch.tensor(l20s)
+
+
+    def prox(self, grad_loss_noise, lam):
+        '''
+        Computes the proximal operator of the 1/2-norm of the gradient of the
+        adversarial loss wrt current noise.
+        '''
+
+        lam = lam[:, None, :, :]
+        sh = list(grad_loss_noise.shape)
+        lam = lam.expand(*sh)
+
+        p_lam = (54 ** (1 / 3) / 4) * lam ** (2 / 3)
+
+        mask1 = (grad_loss_noise > p_lam)
+        mask2 = (torch.abs(grad_loss_noise) <= p_lam)
+        mask3 = (grad_loss_noise < -p_lam)
+        mask4 = mask1 + mask3
+
+        phi_lam_x = torch.arccos((lam / 8) * (torch.abs(grad_loss_noise) / 3)
+                                 ** (-1.5))
+
+        grad_loss_noise[mask4] = ((2 / 3) * torch.abs(grad_loss_noise[mask4])
+                                  * (1 + torch.cos((2 * math.pi) / 3
+                                  - (2 * phi_lam_x[mask4]) / 3))).to(torch.float32)
+        grad_loss_noise[mask3] = -grad_loss_noise[mask3]
+        grad_loss_noise[mask2] = 0
+
+        return grad_loss_noise

vlm_eval/attacks/iht.py

@@ -0,0 +1,97 @@
+# Code taken and adapted from https://github.com/wagnermoritz/GSE
+
+import torch
+from vlm_eval.attacks.attack import Attack
+import math
+
+class IHT(Attack):
+
+    def __init__(self, model, targeted=False, img_range=(0, 1), steps=100, prox='hard',ver=False, lam=5e-5, mask_out='none',stepsize=0.015,eps=4./255.):
+        super().__init__(model, targeted=targeted, img_range=img_range)
+        self.steps = steps
+        self.stepsize = stepsize
+        self.ver = ver
+        self.lam = lam
+        self.eps = eps
+        if mask_out != 'none':
+            self.mask_out = mask_out
+        else:
+            self.mask_out = None
+        if prox == 'hard':
+            self.Prox = self.hardprox
+        else:
+            raise NotImplementedError
+
+
+
+    def _set_mask(self, data):
+        mask = torch.ones_like(data)
+        if self.mask_out == 'context':
+            mask[:, :-1, ...] = 0
+        elif self.mask_out == 'query':
+            mask[:, -1, ...] = 0
+        elif isinstance(self.mask_out, int):
+            mask[:, self.mask_out, ...] = 0
+        elif self.mask_out is None:
+            pass
+        else:
+            raise NotImplementedError(f'Unknown mask_out: {self.mask_out}')
+        return mask
+    
+    def __call__(self, img):
+        
+        for param in self.model.model.parameters():
+            param.requires_grad = False
+
+        img = img.to(self.device)
+        mask_out = self._set_mask(img)       
+        x = torch.zeros_like(img) # perturbation to optimize
+        z = x.clone() # used for FISTA extrapolation
+        t = 1
+        if self.ver:
+            print('')
+
+        for i in range(self.steps):
+                # compue gradient
+            x.requires_grad = True
+            loss = self.model(img + x).sum() if self.targeted else -self.model(img + x).sum()
+            loss.backward()
+            x_grad = x.grad.data * mask_out
+            x = x.detach()
+
+            if self.ver and i % 20 == 0:
+                print(f'Iteration: {i+1}, Loss: {loss}\n', end='')
+
+                # FISTA update
+            with torch.no_grad():
+                t_ = .5 * (1 + math.sqrt(1 + 4 * t ** 2))
+                alpha = (t - 1) / t_
+                t = t_
+                z_ = self.Prox(x=x - self.stepsize * x_grad, 
+                               lam=self.lam * self.stepsize, 
+                               img=img, 
+                               eps=self.eps
+                )
+                x = z_ + alpha * (z_ - z)
+                x = torch.clamp(x,-self.eps,self.eps)
+                z = z_.clone()
+                x = torch.clamp(img + x, *self.img_range) - img
+
+        if self.ver:
+            print('')
+        print(f"L0 pert norm: {x.norm(p=0)}")
+        
+        return (img + x * mask_out).detach(), x.norm(p=0).item()
+
+    def hardprox(self, x, lam, img, eps):
+        '''
+        Computes the hard thresholding proximal operator of the the
+        perturbation x.
+
+        :x:   Perturbation after gradient descent step.
+        :lam: Regularization parameter.
+        '''
+        x_proj = torch.clamp(x,-eps,eps)
+        x_temp = torch.clamp(img + x_proj,*self.img_range)
+        x_proj = x_temp - img
+        return torch.where(x ** 2 - (x_proj - x) ** 2 > 2 * lam, x_proj, 0)

vlm_eval/attacks/pgd.py

@@ -0,0 +1,88 @@
+# Code taken from https://github.com/chs20/RobustVLM/tree/main
+import torch
+from vlm_eval.attacks.utils import project_perturbation, normalize_grad
+
+
+class PGD:
+    """
+    Minimize or maximize given loss
+    """
+
+    def __init__(self, forward, norm, eps, mode='min', mask_out='context', image_space=True):
+        self.model = forward
+
+        self.norm = norm
+        self.eps = eps
+        self.momentum = 0.9
+
+        self.mode = mode
+        self.mask_out = mask_out
+        self.image_space = image_space
+
+    def perturb(self, data_clean, iterations, stepsize, perturbation=None, verbose=False, return_loss=False):
+        if self.image_space:
+            # make sure data is in image space
+            assert torch.max(data_clean) < 1. + 1e-6 and torch.min(data_clean) > -1e-6 # todo
+
+        if perturbation is None:
+            perturbation = torch.zeros_like(data_clean, requires_grad=True)
+        mask = self._set_mask(data_clean)
+        velocity = torch.zeros_like(data_clean)
+        for i in range(iterations):
+            perturbation.requires_grad_()
+            with torch.enable_grad():
+                loss = self.model(data_clean + perturbation)
+                # print 10 times in total and last iteration
+                if verbose and (i % (iterations // 10 + 1) == 0 or i == iterations - 1):
+                    print(f'[iteration] {i} [loss] {loss.item()}')
+
+            with torch.no_grad():
+                gradient = torch.autograd.grad(loss, perturbation)[0]
+                gradient = mask * gradient
+                if gradient.isnan().any():  #
+                    print(f'attention: nan in gradient ({gradient.isnan().sum()})')  #
+                    gradient[gradient.isnan()] = 0.
+                # normalize
+                gradient = normalize_grad(gradient, p=self.norm)
+                # momentum
+                velocity = self.momentum * velocity + gradient
+                velocity = normalize_grad(velocity, p=self.norm)
+                # update
+                if self.mode == 'min':
+                    perturbation = perturbation - stepsize * velocity
+                elif self.mode == 'max':
+                    perturbation = perturbation + stepsize * velocity
+                else:
+                    raise ValueError(f'Unknown mode: {self.mode}')
+                # project
+                perturbation = project_perturbation(perturbation, self.eps, self.norm)
+                if self.image_space:
+                    perturbation = torch.clamp(
+                        data_clean + perturbation, 0, 1
+                    ) - data_clean  # clamp to image space
+                    assert torch.max(data_clean + perturbation) < 1. + 1e-6 and torch.min(
+                        data_clean + perturbation
+                    ) > -1e-6
+                assert not perturbation.isnan().any()
+
+                # assert (ctorch.compute_norm(perturbation, p=self.norm) <= self.eps + 1e-6).all()
+        # todo return best perturbation
+        # problem is that model currently does not output expanded loss
+        if return_loss:
+            return data_clean + perturbation.detach(), loss
+        else:
+            return data_clean + perturbation.detach()
+
+    def _set_mask(self, data):
+        mask = torch.ones_like(data)
+        if self.mask_out == 'context':
+            mask[:, :-1, ...] = 0
+        elif self.mask_out == 'query':
+            mask[:, -1, ...] = 0
+        elif isinstance(self.mask_out, int):
+            mask[:, self.mask_out, ...] = 0
+        elif self.mask_out is None:
+            pass
+        else:
+            raise NotImplementedError(f'Unknown mask_out: {self.mask_out}')
+        return mask

vlm_eval/attacks/pgd0.py

@@ -0,0 +1,131 @@
+# Code taken and adapted from https://github.com/wagnermoritz/GSE
+
+from vlm_eval.attacks.attack import Attack
+import torch
+import numpy as np
+
+class PGD0(Attack):
+    def __init__(self, model, *args, img_range=(0, 1), k=5000, n_restarts=1,
+                 targeted=False, iters=200, stepsize=120000/255.0, eps=4./255.,ver=False,mask_out='none',**kwargs):
+        '''
+        Implementation of the PGD0 attack https://arxiv.org/pdf/1909.05040
+        Author's implementation: https://github.com/fra31/sparse-imperceivable-attacks/tree/master
+        Addapted from: https://github.com/wagnermoritz/GSE/tree/main
+
+        args:
+        model:         Callable, PyTorch classifier.
+        img_range:     Tuple of ints/floats, lower and upper bound of image
+                       entries.
+        targeted:      Bool, given label is used as a target label if True.
+        k:             Int, sparsity parameter.
+        n_restarts:    Int, number of restarts from random perturbation.
+        iters:         Int, number of gradient descent steps per restart.
+        stepsize:      Float, step size for gradient descent.
+        '''
+        super().__init__(model, img_range=img_range, targeted=targeted)
+        self.k = k
+        self.n_restarts = n_restarts
+        self.eps = eps
+        self.iters = iters
+        self.stepsize = stepsize
+        if mask_out != 'none':
+            self.mask_out = mask_out
+        else:
+            self.mask_out = None
+        self.ver = ver
+
+    def _set_mask(self, data):
+        mask = torch.ones_like(data)
+        if self.mask_out == 'context':
+            mask[:, :-1, ...] = 0
+        elif self.mask_out == 'query':
+            mask[:, -1, ...] = 0
+        elif isinstance(self.mask_out, int):
+            mask[:, self.mask_out, ...] = 0
+        elif self.mask_out is None:
+            pass
+        else:
+            raise NotImplementedError(f'Unknown mask_out: {self.mask_out}')
+        return mask
+
+
+    def __call__(self, x, *args, **kwargs):
+        '''
+        Perform the PGD_0 attack on a batch of images x.
+
+        args:
+        x:   Tensor of shape [B, C, H, W], batch of images.
+        y:   Tensor of shape [B], batch of labels.
+
+        Returns a tensor of the same shape as x containing adversarial examples
+        '''
+
+        for param in self.model.model.parameters():
+            param.requires_grad = False
+        
+        mask_out = self._set_mask(x)
+        x = x.to(self.device)
+        B, C, H, W = x.shape[1], x.shape[3], x.shape[4], x.shape[5]
+
+        for _ in range(self.n_restarts):
+            if not len(x):
+                break
+            eps = torch.full_like(x, self.eps)
+            lb, ub = torch.maximum(-eps, -x),torch.minimum(eps, 1.0 - x) #self.img_range[0] - x, self.img_range[1] - x
+            pert = (torch.clamp(x + (ub - lb) * torch.rand_like(x) + lb, *self.img_range) - x).view(B, C, H, W) * mask_out.view(B, C, H, W)
+            pert = self.project_L0(pert, lb, ub) # pert is of the shape (B, C, H, W)
+
+            for _ in range(self.iters):
+                pert.requires_grad = True
+                loss = self.lossfn(x=x, pert=pert.view(*x.shape), mask_out=mask_out)
+                loss.backward()
+
+                if self.ver and _ % 20 == 0:
+                    print(f"Loss: {loss}, Iter: {_}")
+                
+                grad = pert.grad.data.view(B,C,H,W) * mask_out.view(B, C, H, W) # shape (B, C, H, W)
+                with torch.no_grad():
+                    grad /= grad.abs().sum(dim=(1,2,3), keepdim=True) + 1e-10
+                    pert += (torch.rand_like(x) - .5).view(B, C, H, W) * 1e-12 - self.stepsize * grad
+                    pert = self.project_L0(pert, lb, ub)
+        
+        return (x + pert.view(*x.shape) * mask_out).detach()
+    
+
+    def project_L0_sigma(self, pert, sigma, kappa, x_orig):
+
+        B, C, H, W = pert.shape
+        x = torch.clone(pert)
+        p1 = (1.0 / torch.maximum(1e-12, sigma) * (x_orig > 0).float()) + \
+             (1e12 * (x_orig == 0).float())
+        p2 = (1.0 / torch.maximum(torch.tensor(1e-12), sigma)) * \
+             (1.0 / torch.maximum(torch.tensor(1e-12), x_orig) - 1) * \
+             (x_orig > 0).float() + 1e12 * (x_orig == 0).float() + 1e12 * (sigma == 0).float()
+        lmbd_l = torch.maximum(-kappa, torch.amax(-p1, dim=1, keepdim=True))
+        lmbd_u = torch.minimum(kappa, torch.amin(p2, dim=1, keepdim=True)) 
+
+        lmbd_unconstr = torch.sum((pert - x_orig) * sigma * x_orig, dim=1, keepdim=True) / torch.clamp(torch.sum((sigma * x_orig) ** 2, dim=1, keepdim=True), min=1e-12)
+        lmbd = torch.maximum(lmbd_l, torch.minimum(lmbd_unconstr, lmbd_u))
+        return 0
+
+
+    def project_L0(self, pert, lb, ub):
+        '''
+        Project a batch of perturbations such that at most self.k pixels
+        are perturbed and componentwise there holds lb <= pert <= ub.
+        '''
+        
+        B, C, H, W = pert.shape # Here, pert is of the shape B, C, H, W
+        p1 = torch.sum(pert ** 2, dim=1)
+        p2 = torch.clamp(torch.minimum(ub.view(B, C, H, W) - pert, pert - lb.view(B, C, H, W)), 0)
+        p2 = torch.sum(p2 ** 2, dim=1)
+        p3 = torch.topk(-1 * (p1 - p2).view(p1.size(0), -1), k=H*W-self.k, dim=-1)[1] 
+        pert = torch.maximum(torch.minimum(pert, ub.view(B, C, H, W)), lb.view(B, C, H, W))
+        pert[torch.arange(0, B).view(-1, 1), :, p3//W, p3%H] = 0  
+        return pert
+        
+    def lossfn(self, x, pert, mask_out):
+        '''
+        Compute the loss at x.
+        '''
+        return (2 * self.targeted - 1) * self.model(x + pert * mask_out).sum()

vlm_eval/attacks/saif.py

@@ -0,0 +1,143 @@
+# Code adapted from https://github.com/wagnermoritz/GSE
+
+from vlm_eval.attacks.attack import Attack
+import torch
+import math
+import time
+
+class SAIF(Attack):
+    def __init__(self, model, *args, targeted=False, img_range=(-1, 1), steps=200,
+                 r0=1, ver=False, k=10000, eps=16./255., mask_out='none', **kwargs):
+        '''
+        Adapted from: https://github.com/wagnermoritz/GSE/tree/main
+        Implementation of the sparse Frank-Wolfe attack SAIF
+        https://arxiv.org/pdf/2212.07495.pdf
+
+        args:
+        model:         Callable, PyTorch classifier.
+        img_range:     Tuple of ints/floats, lower and upper bound of image
+                       entries.
+        targeted:      Bool, given label is used as a target label if True.
+        steps:         Int, number of FW iterations.
+        r0:            Int, parameter for step size computation.
+        ver:           Bool, print progress if True.
+        '''
+        super().__init__(model, targeted=targeted, img_range=img_range)
+        self.steps = steps
+        self.r0 = r0
+        self.loss_fn = torch.nn.CrossEntropyLoss()
+        self.ver = ver
+        self.k = k
+        self.eps = eps
+        if mask_out != 'none':
+            self.mask_out = mask_out
+        else:
+            self.mask_out = None
+
+    def _set_mask(self, data):
+        mask = torch.ones_like(data)
+        if self.mask_out == 'context':
+            mask[:, :-1, ...] = 0
+        elif self.mask_out == 'query':
+            mask[:, -1, ...] = 0
+        elif isinstance(self.mask_out, int):
+            mask[:, self.mask_out, ...] = 0
+        elif self.mask_out is None:
+            pass
+        else:
+            raise NotImplementedError(f'Unknown mask_out: {self.mask_out}')
+        return mask
+
+    def __call__(self, x):
+        '''
+        Perform the  attack on a batch of images x.
+
+        args:
+        x:   Tensor of shape [B, C, H, W], batch of images.
+        k:   Int, sparsity parameter,
+        eps: Float, perturbation magnitude parameter.
+
+        Returns a tensor of the same shape as x containing adversarial examples.
+        '''
+        assert x.shape[0] == 1, "Only support batch size 1 for now"
+
+
+
+        for param in self.model.model.parameters():
+            param.requires_grad = False
+
+        B, C, H, W = x.shape[1], x.shape[3], x.shape[4], x.shape[5]
+        x = x.to(self.device)
+        batchidx = torch.arange(B).view(-1, 1)
+
+        mask_out = self._set_mask(x)
+        # compute p_0 and s_0
+        x_ = x.clone()
+        x_.requires_grad = True
+        out = self.model(x_)
+        loss = -out.sum() if not self.targeted else out.sum()
+        x__grad = torch.autograd.grad(loss, [x_])[0].detach() * mask_out
+        p = -self.eps * x__grad.sign()
+        p = p.detach().half()
+        ksmallest = torch.topk(-x__grad.view(B, -1), self.k, dim=1)[1]
+        ksmask = torch.zeros((B, C * H * W), device=self.device)
+        ksmask[batchidx, ksmallest] = 1
+        s = torch.logical_and(ksmask.view(*x.shape), x__grad < 0).float()
+        s = s.detach().half()
+
+        r = self.r0
+
+
+        for t in range(self.steps):
+            if self.ver:
+                print(f'\r Iteration {t+1}/{self.steps}', end='')
+            p.requires_grad = True
+            s.requires_grad = True
+
+            D = self.Loss_fn(x, s, p, mask_out)
+            D.backward()
+
+            mp = p.grad * mask_out
+            ms = s.grad * mask_out
+            with torch.no_grad():
+                # inf-norm LMO
+                v = (-self.eps * mp.sign()).half()
+
+                # 1-norm LMO
+                ksmallest = torch.topk(-ms.view(B, -1), self.k, dim=1)[1]
+                ksmask = torch.zeros((B, C * H * W), device=self.device)
+                ksmask[batchidx, ksmallest] = 1
+                ksmask = ksmask.view(*x.shape) * mask_out
+                z = torch.logical_and(ksmask, ms < 0).float().half()
+                # update stepsize until primal progress is made
+                mu = 1 / (2 ** r * math.sqrt(t + 1))
+                progress_condition = (self.Loss_fn(x, s + mu * (z - s), p + mu * (v - p), mask_out)
+                    > D)
+
+                while progress_condition:
+                    r += 1
+                    if r >= 50:
+                        break
+                    mu = 1 / (2 ** r * math.sqrt(t + 1))
+                    progress_condition = (self.Loss_fn(x, s + mu * (z - s), p + mu * (v - p), mask_out)
+                    > D)
+
+
+                p = p + mu * (v - p)
+                s = s + mu * (z - s)
+
+                x_adv = torch.clamp(x + p, *self.img_range)
+                p = x_adv - x
+
+            if self.ver and t % 10 == 0:
+                print(f" Loss: {D}")
+        if self.ver:
+            print('')
+        return (x + s * p * mask_out).detach(), torch.norm(s*p,p=0).item()
+
+    def Loss_fn(self, x, s, p, mask_out):
+        out = self.model(x + s * p * mask_out).sum()
+        if self.targeted:
+            return out
+        else:
+            return -out

vlm_eval/attacks/sparsers.py

@@ -0,0 +1,164 @@
+# Code taken and adapted from https://github.com/wagnermoritz/GSE
+from vlm_eval.attacks.attack import Attack
+import torch
+
+class SparseRS(Attack):
+    def __init__(self, model, *args, targeted=False, img_range=(-1, 1),
+                 n_queries=10000, k=100, n_restarts=10, alpha_init=0.8, mask_out='none',**kwargs):
+        '''
+        Implementation of the L0 variant SparseRS https://arxiv.org/abs/2006.12834
+        Authors' implementation: https://github.com/fra31/sparse-rs
+        Adapted from: https://github.com/wagnermoritz/GSE/tree/main
+        
+        args:
+        model:         Callable, PyTorch classifier.
+        targeted:      Bool, given label is used as a target label if True.
+        img_range:     Tuple of ints/floats, lower and upper bound of image
+                       entries.
+        n_queries:     Int, max number of queries to the model
+        k:             Int, initial sparsity parameter
+        n_restarts:    Int, number of restarts with random initialization
+        alpha_init:    Float, inital value for alpha schedule
+        '''
+        super().__init__(model, targeted=targeted, img_range=img_range)
+        self.n_queries = n_queries
+        self.k = k
+        self.n_restarts = n_restarts
+        self.alpha_init = alpha_init
+        if mask_out != 'none':
+            self.mask_out = mask_out
+        else:
+            self.mask_out = None
+
+    def _set_mask(self, data):
+        mask = torch.ones_like(data)
+        if self.mask_out == 'context':
+            mask[:, :-1, ...] = 0
+        elif self.mask_out == 'query':
+            mask[:, -1, ...] = 0
+        elif isinstance(self.mask_out, int):
+            mask[:, self.mask_out, ...] = 0
+        elif self.mask_out is None:
+            pass
+        else:
+            raise NotImplementedError(f'Unknown mask_out: {self.mask_out}')
+        return mask
+
+
+    def __call__(self, x, *args, **kwargs):
+        '''
+        Perform SparseRS L0 on a batch of images x with corresponding labels y.
+
+        args:
+        x:   Tensor of shape [B, C, H, W], batch of images.
+        y:   Tensor of shape [B], batch of labels.
+
+        Returns a tensor of the same shape as x containing adversarial examples
+        '''
+
+        for param in self.model.model.parameters():
+            param.requires_grad = False
+
+        torch.random.manual_seed(0)
+        torch.cuda.random.manual_seed(0)
+        x = x.to(self.device)
+
+        with torch.no_grad():
+            for _ in range(self.n_restarts):
+                if len(x) == 0:
+                    break
+
+                x_adv = self.__perturb(x.clone())
+
+        return x_adv.detach()
+    
+
+    def __perturb(self, x):
+        '''
+        Perform the attack from a random starting point.
+        '''
+        mask_out = self._set_mask(x)
+        B, C, H, W = x.shape[1], x.shape[3], x.shape[4], x.shape[5]
+        batchidx = torch.arange(B, device=self.device).view(-1, 1)
+        result = x.clone().view(B, C, H, W)
+
+        # M: set of perturbed pixel indices, U_M: set of unperturbed pixel indices
+        batch_randperm = torch.rand(B, H * W, device=self.device).argsort(dim=1)
+        M = batch_randperm[:, :self.k]
+        U_M = batch_randperm[:, self.k:]
+        result[batchidx, :, M//W, M%H] = self.__sampleDelta(B, C, self.k)
+
+        best_loss = self.__lossfn(result.view(*x.shape))
+
+        for i in range(1, self.n_queries):
+            if B == 0:
+                break
+            # reset k_i currently perturbed pixels and perturb k_i new pixels
+            k_i = max(int(self.__alphaSchedule(i) * self.k), 1)
+            A_idx = torch.randperm(self.k, device=self.device)[:k_i]
+            B_idx = torch.randperm(H * W - self.k, device=self.device)[:k_i]
+            A_set, B_set = M[:, A_idx], U_M[:, B_idx]
+
+            z = result.clone()
+            z[batchidx, :, A_set//W, A_set%H] = x.view(B, C, H, W)[batchidx, :, A_set//W, A_set%H]
+            if k_i > 1:
+                z[batchidx, :, B_set//W, B_set%H] = self.__sampleDelta(B, C, k_i)
+            else: # if only one pixel is changed, make sure it actually changes
+                new_color = self.__sampleDelta(B, C, k_i)
+                while (mask := (z[batchidx, :, B_set//W, B_set%H] == new_color).view(B, -1).all(dim=-1)).any():
+                    new_color[mask] = self.__sampleDelta(mask.int().sum().item(), C, k_i)
+                z[batchidx, :, B_set//W, B_set%H] = new_color
+
+            # save perturbations that improved the loss/margin
+            loss = self.__lossfn(z, y)
+            mask = loss < best_loss
+            best_loss[mask] = loss[mask]
+            mask = torch.logical_or(mask, margin < -1e-6)
+            if mask.any():
+                #best_margin[mask] = margin[mask]
+                tmp = result[active]
+                tmp[mask] = z[mask]
+                result[active] = tmp
+                U_M[mask.nonzero().view(-1, 1), B_idx] = A_set[mask]
+                M[mask.nonzero().view(-1, 1), A_idx] = B_set[mask]
+            
+            # stop working on successful adv examples
+            mask = best_margin < 0
+            if mask.any():
+                mask = torch.logical_not(mask)
+                active[active.clone()] = mask
+                x, y, z, M, U_M = x[mask], y[mask], z[mask], M[mask], U_M[mask]
+                best_margin, best_loss = best_margin[mask], best_loss[mask]
+                B = len(y)
+                batchidx = torch.arange(B, device=self.device).view(-1, 1)
+
+        return result
+
+
+    def __sampleDelta(self, B, C, k):
+        '''
+        Sample k-pixel perturbations for B images. Each pixel is assigned a
+        random corner in the C-dimensional cube defined by self.img_range.
+        '''
+        fac = self.img_range[1] - self.img_range[0]
+        return self.img_range[0] + fac * torch.randint(0, 1, [B, k, C],
+                                                       dtype=torch.float,
+                                                       device=self.device)
+    
+
+    def __alphaSchedule(self, iteration):
+        '''
+        Update number of pixels to perturb based in the current iteration.
+        '''
+        iteration = int(iteration / self.n_queries * 10000)
+        factors = [1, 2, 4, 5, 6, 8, 10, 12, 15, 20]
+        alpha_schedule = [10, 50, 200, 500, 1000, 2000, 4000, 6000, 8000]
+        idx = bisect.bisect_left(alpha_schedule, iteration)
+        return self.alpha_init / factors[idx]
+    
+
+    def __lossfn(self, x):
+        '''
+        Compute the loss depending on self.targeted.
+        '''
+        return self.model(x).sum() if self.targeted else -self.model(x).sum()

vlm_eval/attacks/strattack.py

@@ -0,0 +1,229 @@
+# Code taken and adapted from https://github.com/wagnermoritz/GSE
+
+from vlm_eval.attacks.attack import Attack
+import torch
+import math
+import torch.nn.functional as F
+
+class StrAttack(Attack):
+    def __init__(self, model, *args, targeted=False, img_range=(0, 1), kappa=0,
+                 max_iter=100, ver=False, search_steps=2, max_c=1e10, rho=1, mask_out='none',
+                 c=2.5, retrain=False, **kwargs):
+        '''
+        Implementation of StrAttack: https://arxiv.org/abs/1808.01664
+        Adapted from https://github.com/KaidiXu/StrAttack
+
+        args:
+        model:         Callable, PyTorch classifier.
+        targeted:      Bool, given label is used as a target label if True.
+        img_range:     Tuple of ints/floats, lower and upper bound of image
+                       entries.
+        max_iter:      Int, number of iterations.
+        ver:           Bool, print progress if True.
+        search_steps:  Int, number of binary search steps.
+        max_c:         Float, upper bound for regularizaion parameter.
+        rho:           Float, ADMM parameter.
+        c:             Float, initial regularization parameter.
+        '''
+        super().__init__(model, targeted=targeted, img_range=img_range)
+        self.max_iter = max_iter
+        self.ver = ver
+        self.search_steps = search_steps
+        self.max_c = max_c
+        self.rho = rho
+        self.c = c
+        self.retrain = retrain
+        if mask_out != 'none':
+            self.mask_out = mask_out
+        else:
+            self.mask_out = None
+
+    def _set_mask(self, data):
+        mask = torch.ones_like(data)
+        if self.mask_out == 'context':
+            mask[:, :-1, ...] = 0
+        elif self.mask_out == 'query':
+            mask[:, -1, ...] = 0
+        elif isinstance(self.mask_out, int):
+            mask[:, self.mask_out, ...] = 0
+        elif self.mask_out is None:
+            pass
+        else:
+            raise NotImplementedError(f'Unknown mask_out: {self.mask_out}')
+        return mask
+
+    def __call__(self, imgs, *args, **kwargs):
+        '''
+        Perform StrAttack on a batch of images x with corresponding labels y.
+
+        args:
+        x:   Tensor of shape [B, C, H, W], batch of images.
+
+        Returns a tensor of the same shape as x containing adversarial examples
+        '''
+
+        for param in self.model.model.parameters():
+            param.requires_grad = False
+
+        c_ = self.c
+        imgs = imgs.to(self.device)
+        sh = imgs.shape
+        batch_size = sh[1]
+        mask_out = self._set_mask(imgs)
+
+        alpha, tau, gamma = 5, 2, 1
+        eps = torch.full_like(imgs, 1.0) * mask_out
+        # 16 for imagenet, 2 for CIFAR and MNIST
+        filterSize = 8 if sh[-1] > 32 else 2
+        stride = filterSize
+        # convolution kernel used to compute norm of each group
+        slidingM = torch.ones((1, sh[3], filterSize, filterSize), device=self.device)
+
+        cs = torch.ones(batch_size, device=self.device) * c_
+        lower_bound = torch.zeros(batch_size)
+        upper_bound = torch.ones(batch_size) * self.max_c
+
+        o_bestl2 = torch.full_like(torch.randn(batch_size), 1e10, dtype=torch.float)
+        o_bestscore = torch.full_like(o_bestl2, -1, dtype=torch.float)
+        o_bestattack = imgs.clone()
+        o_besty = torch.ones_like(imgs)
+
+        for step in range(self.search_steps):
+
+            bestl2 = torch.full_like(o_bestl2, 1e10, dtype=torch.float)
+            bestscore = torch.full_like(o_bestl2, -1, dtype=torch.float)
+
+            z, v, u, s = (torch.zeros_like(imgs) for _ in range(4))
+
+            for iter_ in range(self.max_iter):
+                if (not iter_%10 or iter_ == self.max_iter - 1) and self.ver:
+                    print(f'\rIteration: {iter_+1}/{self.max_iter}, ' +
+                          f'Search Step: {step+1}/{self.search_steps}', end='')
+
+                # first update step (7) / Proposition 1
+                delta = self.rho / (self.rho + 2 * gamma) * (z - u / self.rho)
+
+                b = (z - s / self.rho) * mask_out
+                tmp = torch.minimum(self.img_range[1] - imgs, eps)
+                w = torch.where(b.view(*sh) > tmp.view(*sh), tmp, b) # creating issue (1x5x'5'x3x224x224 instead of 1x5x1x3x224x224)
+                tmp = torch.maximum(self.img_range[0] - imgs, -eps)
+                w = torch.where(b.view(*sh) < tmp.view(*sh), tmp, w)
+                
+                c = z - v / self.rho
+                cNorm = torch.sqrt(F.conv2d(c.view(sh[1], sh[3], sh[4], sh[5]) ** 2, slidingM, stride=stride))
+                cNorm = torch.where(cNorm == 0, torch.full_like(cNorm, 1e-12), cNorm)
+                cNorm = F.interpolate(cNorm, scale_factor=filterSize)
+                y = torch.clamp((1 - tau / (self.rho * cNorm.unsqueeze(0).unsqueeze(3))), 0) * c
+                
+                # second update step (8) / equation (15)
+                z_grads = self.__get_z_grad(imgs, z.clone(), cs)
+                eta = alpha * math.sqrt(iter_ + 1)
+                coeff = (1 / (eta + 3 * self.rho))
+                z = coeff * (eta * z + self.rho * (delta + w + y) + u + s + v - z_grads)
+
+                # third update step (9)
+                u = u + self.rho * (delta - z) * mask_out
+                v = v + self.rho * (y - z) * mask_out
+                s = s + self.rho * (w - z) * mask_out
+                # get info for binary search
+                x = imgs + y * mask_out
+                l2s = torch.sum((z ** 2).reshape(z.size(1), -1), dim=-1)
+                
+                for i, (l2, x_) in enumerate(zip(l2s, x.squeeze(0))):
+                    if l2 < bestl2[i]:
+                        bestl2[i] = l2
+                    if l2 < o_bestl2[i]:
+                        o_bestl2[i] = l2
+                        o_bestattack[:,i] = x_.detach().unsqueeze(0).clone()
+                        o_besty[:,i] = y[:,i]
+            for i in range(batch_size):
+                
+                lower_bound[i] = max(lower_bound[i], cs[i])
+                if upper_bound[i] < 1e9:
+                    cs[i] = (lower_bound[i] + upper_bound[i]) / 2
+                else:
+                    cs[i] *= 5
+
+        del v, u, s, z_grads, w, tmp
+        
+        if self.retrain:
+            cs = torch.full_like(o_bestl2, 5.0, dtype=torch.float)
+            zeros = torch.zeros_like(imgs)
+
+            for step in range(8):
+                bestl2 = torch.full_like(cs, 1e10, dtype=torch.float, device=self.device)
+                bestscore = torch.full_like(cs, -1, dtype=torch.float, device=self.device)
+
+                Nz = o_besty[o_besty != 0]
+                e0 = torch.quantile(Nz.abs(), 0.03)
+                A2 = torch.where(o_besty.abs() <= e0, 0, 1)
+                z1 = o_besty
+                u1 = torch.zeros_like(imgs)
+                tmpc = self.rho / (self.rho + gamma / 100)
+
+                for j in range(100):
+                    if self.ver and not j % 10:
+                        print(f'\rRetrain iteration: {step+1}/8, ' +
+                              f'Search Step: {j+1}/200', end='')
+
+                    tmpA = (z1 - u1) * tmpc
+                    tmpA1 = torch.where(o_besty.abs() <= e0, zeros, tmpA)
+                    cond = torch.logical_and(tmpA >
+                                             torch.minimum(self.img_range[1] - imgs, eps),
+                                             o_besty.abs() > e0)
+                    tmpA2 = torch.where(cond, torch.minimum(self.img_range[1] - imgs, eps),
+                                        tmpA1)
+                    cond = torch.logical_and(tmpA <
+                                             torch.maximum(self.img_range[0] - imgs, -eps),
+                                             o_besty.abs() > e0)
+                    deltA = torch.where(cond, torch.maximum(self.img_range[0] - imgs, -eps),
+                                        tmpA2)
+                    
+                    x = imgs + deltA * mask_out
+                    grad = self.__get_z_grad(imgs, deltA, cs)
+
+                    stepsize = 1 / (alpha + 2 * self.rho)
+                    z1 = stepsize * (alpha * z1 * self.rho
+                                     * (deltA + u1) - grad * A2)
+                    u1 = u1 + deltA - z1
+
+                    for i, (l2, x_) in enumerate(zip(l2s, x.squeeze(0))):
+                        if l2 < bestl2[i]:
+                            bestl2[i] = l2
+                            #bestscore[i] = asc
+                        if l2 < o_bestl2[i]:
+                            o_bestl2[i] = l2
+                            #o_bestscore[i] = asc
+                            o_bestattack[:,i] = x_.detach().unsqueeze(0).clone()
+                            o_besty[i] = deltA[i]
+
+
+                for i in range(batch_size):
+                    if (bestscore[i] != -1 and bestl2[i] == o_bestl2[i]):
+                        upper_bound[i] = min(upper_bound[i], cs[i])
+                        if upper_bound[i] < 1e9:
+                            cs[i] = (lower_bound[i] + upper_bound[i]) / 2
+
+                    else:
+                        lower_bound[i] = max(lower_bound[i], cs[i])
+                        if upper_bound[i] < 1e9:
+                            cs[i] = (lower_bound[i] + upper_bound[i]) / 2
+                        else:
+                            cs[i] *= 5
+
+        if self.ver:
+            print('')
+        
+        return (o_bestattack * mask_out).detach()
+
+
+    def __get_z_grad(self, imgs, z, cs):
+        '''
+        Compute and return gradient of loss wrt. z.
+        '''
+        z.requires_grad = True
+        tmp = self.model(z + imgs).sum() if self.targeted else -self.model(z + imgs).sum()
+        loss = torch.mean(cs.to(self.device) * tmp)
+        z_grad_data = torch.autograd.grad(loss, [z])[0].detach()
+        z.detach_()
+        return z_grad_data

vlm_eval/attacks/utils.py

@@ -0,0 +1,52 @@
+import torch
+import torch.nn.functional as F
+import collections.abc as container_abcs
+
+# Code taken from https://github.com/chs20/RobustVLM/tree/main
+# some parts of this code are adapted from
+# https://github.com/M4xim4l/InNOutRobustness/blob/main/utils/adversarial_attacks/utils.py
+
+def project_perturbation(perturbation, eps, norm):
+    if norm in ['inf', 'linf', 'Linf']:
+        pert_normalized = torch.clamp(perturbation, -eps, eps)
+        return pert_normalized
+    elif norm in [2, 2.0, 'l2', 'L2', '2']:
+        pert_normalized = torch.renorm(perturbation, p=2, dim=0, maxnorm=eps)
+        return pert_normalized
+    else:
+        raise NotImplementedError(f'Norm {norm} not supported')
+
+
+def normalize_grad(grad, p):
+    if p in ['inf', 'linf', 'Linf']:
+        return grad.sign()
+    elif p in [2, 2.0, 'l2', 'L2', '2']:
+        bs = grad.shape[0]
+        grad_flat = grad.view(bs, -1)
+        grad_normalized = F.normalize(grad_flat, p=2, dim=1)
+        return grad_normalized.view_as(grad)
+
+
+def L1_norm(x, keepdim=False):
+    z = x.abs().view(x.shape[0], -1).sum(-1)
+    if keepdim:
+        z = z.view(-1, *[1]*(len(x.shape) - 1))
+    return z
+
+def L2_norm(x, keepdim=False):
+    z = (x ** 2).view(x.shape[0], -1).sum(-1).sqrt()
+    if keepdim:
+        z = z.view(-1, *[1]*(len(x.shape) - 1))
+    return z
+
+def L0_norm(x):
+    return (x != 0.).view(x.shape[0], -1).sum(-1)
+
+def zero_gradients(x):
+    if isinstance(x, torch.Tensor):
+        if x.grad is not None:
+            x.grad.detach_()
+            x.grad.zero_()
+    elif isinstance(x, container_abcs.Iterable):
+        for elem in x:
+            zero_gradients(elem)

vlm_eval/clip_classification.py

@@ -0,0 +1,160 @@
+# Code adapted from https://github.com/openai/CLIP/blob/main/
+from transformers import CLIPProcessor, CLIPModel
+import argparse
+import torch
+from torch.utils.data import DataLoader
+from tqdm import tqdm
+from datasets_classes_templates import data_seeds
+import numpy as np
+from datetime import datetime
+
+def zeroshot_classifier(classnames, templates, processor, model):
+    with torch.no_grad():
+        zeroshot_weights = []
+        for classname in tqdm(classnames):
+            texts = [template.format(classname) for template in templates] #format with class
+            text_inputs = processor(text=texts, return_tensors="pt", padding=True, truncation=True).to('cuda')
+            class_embeddings = model.get_text_features(text_inputs['input_ids']) #embed with text encoder
+            class_embeddings /= class_embeddings.norm(dim=-1, keepdim=True)
+            class_embedding = class_embeddings.mean(dim=0)
+            class_embedding /= class_embedding.norm()
+            zeroshot_weights.append(class_embedding)
+        zeroshot_weights = torch.stack(zeroshot_weights, dim=1).cuda()
+    return zeroshot_weights
+
+def classification_collate_fn(batch):
+    images, labels = zip(*batch)
+    labels = torch.tensor(labels)
+    return images, labels
+
+def accuracy(output, target, topk=(1,)):
+    pred = output.topk(max(topk), 1, True, True)[1].t()
+    correct = pred.eq(target.view(1, -1).expand_as(pred))
+    return [float(correct[:k].reshape(-1).float().sum(0, keepdim=True).cpu().numpy()) for k in topk]
+
+
+def main():
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--data", type=str, default=None, choices=['non_fine_tuned','MS_COCO','medium','base','all'], help='Data on which clip was fine-tuned')
+    parser.add_argument("--dataset", type=str, default="CIFAR10", choices=["CIFAR10", "CIFAR100", "ImageNet", "Caltech101", "Caltech256", "Food101"])
+    parser.add_argument("--method",type=str, default="COCO_CF", choices=['COCO_CF','APGD_1','APGD_4','NONE'])
+    args = parser.parse_args()
+
+    current_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+    results_filename = f'./Results/fine_tuned_clip/zeroshot_image_classification_results_{args.dataset}_{args.data}_{args.method}_{current_time}.txt'
+    with open(results_filename, 'w') as f:
+        f.write(f'Arguments: {args}\n\n')
+
+    if args.data == 'MS_COCO':
+        assert args.method == 'NONE' and args.data == 'MS_COCO', 'Use NONE for method for MS_COCO data'
+
+    imagenet_path = '/software/ais2t/pytorch_datasets/imagenet/' # Fill the path for imagenet here
+
+    if args.dataset == "CIFAR10":
+        from datasets_classes_templates import CIFAR10_CLASSES_TEMPLATES as classes_templates
+        from torchvision.datasets import CIFAR10
+        data = CIFAR10(root='./image_classification_datasets/cifar10/', train=False, download=True)
+    elif args.dataset == "CIFAR100":
+        from datasets_classes_templates import CIFAR100_CLASSES_TEMPLATES as classes_templates
+        from torchvision.datasets import CIFAR100
+        data = CIFAR100(root='./image_classification_datasets/cifar100/', train=False, download=True)
+    elif args.dataset == "ImageNet":
+        from datasets_classes_templates import ImageNet_CLASSES_TEMPLATES as classes_templates
+        from torchvision.datasets import ImageNet
+        data = ImageNet(root=imagenet_path, split='val')
+    elif args.dataset == "Caltech101":
+        torch.manual_seed(42)
+        from datasets_classes_templates import Caltech101_CLASSES_TEMPLATES as classes_templates
+        from torchvision.datasets import Caltech101
+        data = Caltech101(root='./image_classification_datasets/', download=False)
+        train_size = int(0.8 * len(data))  # 80% for training
+        val_size = len(data) - train_size
+        _, data = torch.utils.data.random_split(data, [train_size, val_size])
+    elif args.dataset == "Caltech256":
+        torch.manual_seed(42)
+        from datasets_classes_templates import Caltech256_CLASSES_TEMPLATES as classes_templates
+        from torchvision.datasets import Caltech256
+        data = Caltech256(root='./image_classification_datasets/', download=False)
+        train_size = int(0.8 * len(data))  # 80% for training
+        val_size = len(data) - train_size
+        _, data = torch.utils.data.random_split(data, [train_size, val_size])
+    elif args.dataset == "Food101":
+        from datasets_classes_templates import Food101_CLASSES_TEMPLATES as classes_templates
+        from torchvision.datasets import Food101
+        data = Food101(root='./image_classification_datasets/food101/', download=True, split='test')
+    else:
+        raise NotImplementedError
+
+    print(f'Conducting zero-shot image classification on {args.dataset}')
+
+    device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
+    model_base_path = './fine_tuned_clip_models'
+    processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
+
+    top1_list = []
+    for data_seed in data_seeds:
+        print(f'Conducting zero-shot image classification on {args.data} with seed {data_seed} for the method {args.method}')
+        model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32").to(device)
+        if args.data != 'non_fine_tuned':
+            if args.method != 'NONE':
+                if args.data not in ['all']:
+                    model.load_state_dict(torch.load(f'{model_base_path}/{args.method}/clip_model_dataset_{args.data}_method_{args.method}_num_epochs_20_data_seed_{data_seed}.pt'))
+                else:
+                    model.load_state_dict(torch.load(f'{model_base_path}/{args.method}/clip_model_dataset_{args.data}_method_{args.method}_num_epochs_20.pt'))
+            elif args.method == 'NONE' and args.data == 'MS_COCO':
+                model.load_state_dict(torch.load(f'{model_base_path}/{args.method}/clip_model_dataset_{args.data}_method_{args.method}_num_epochs_20.pt'))
+
+        model.eval()
+
+        data_loader = DataLoader(data, batch_size=128, collate_fn=classification_collate_fn, shuffle=False)
+
+        zeroshot_weights = zeroshot_classifier(classes_templates['classes'],
+                                            classes_templates['templates'],
+                                            processor,
+                                            model
+        )
+
+        with torch.no_grad():
+            top1, top5, n = 0., 0., 0.
+            for i, (images, target) in enumerate(tqdm(data_loader)):
+                target = target.to(device)
+                images = list(images)
+
+                images = processor(images=images, return_tensors="pt").to(device)
+
+                # predict
+                image_features = model.get_image_features(images['pixel_values']).to(device)
+                image_features /= image_features.norm(dim=-1, keepdim=True)
+                logits = 100. * image_features @ zeroshot_weights
+
+                # measure accuracy
+                acc1, acc5 = accuracy(logits, target, topk=(1, 5))
+                top1 += acc1
+                top5 += acc5
+                n += image_features.size(0)
+
+        top1 = (top1 / n) * 100
+        top5 = (top5 / n) * 100
+
+        with open(results_filename, 'a') as f:
+            f.write(f'Seed {data_seed}: Top-1 Accuracy: {top1:.2f}, Top-5 Accuracy: {top5:.2f}\n')
+
+        top1_list.append(top1)
+
+        print(f"Top-1 accuracy: {top1:.2f}")
+        print(f"Top-5 accuracy: {top5:.2f}")
+        print('-'*40)
+
+        if args.method == 'NONE' or args.data in ['MS_COCO','all'] or args.data == 'non_fine_tuned':
+            break
+    top1 = np.asarray(top1_list)
+    print(f'Mean of the top 1 accuracy is {np.mean(top1)}')
+    print(f'Standard deviation of the top 1 accuracy is {np.std(top1)}')
+
+    with open(results_filename, 'a') as f:
+        f.write(f'\nMean Top-1 Accuracy: {np.mean(top1):.2f}\n')
+        f.write(f'Standard Deviation of Top-1 Accuracy: {np.std(top1):.2f}\n')
+
+if __name__ == "__main__":
+    main()

vlm_eval/clip_train.py

@@ -0,0 +1,209 @@
+# Code adapted from https://github.com/ylaxor/clip-like/blob/main/fine-tune-clip.ipynb
+
+from random import seed, shuffle
+from typing import Callable
+import torch
+from tqdm import tqdm
+from transformers import CLIPProcessor, CLIPModel
+from timm.scheduler import CosineLRScheduler
+
+
+
+class ModelTrainer:
+
+    def __init__(self,
+                 model: Callable,
+                 processor: Callable,
+                 data_name: str,
+                 train_data_loader: torch.utils.data.DataLoader,
+                 val_data_loader: torch.utils.data.DataLoader,
+                 num_epochs: int,
+                 learning_rate: float = 5e-5,
+                 weight_decay: float = 1e-3,
+                 device: str = "cuda:0",
+                 save_model: bool = False,
+                 save_model_path: str = "./fine_tuned_clip_models",
+                 data_seed: int = 42,
+                 method="COCO_CF",
+    ) -> None:
+
+        self.model = model
+        self.processor = processor
+        self.data_name = data_name
+        self.train_data_loader = train_data_loader
+        self.val_data_loader = val_data_loader
+        self.num_epochs = num_epochs
+        self.learning_rate = learning_rate
+        self.weight_decay = weight_decay
+        self.device = device
+        self.save_model = save_model
+        self.save_model_path = save_model_path
+        self.data_seed = data_seed
+        self.method = method
+
+        self.optimizer = torch.optim.AdamW(
+            self.model.parameters(),
+            lr=self.learning_rate,
+            weight_decay=self.weight_decay
+        )
+
+
+    def train(self):
+        self.model.train()
+        lr_scheduler = CosineLRScheduler(
+            self.optimizer,
+            t_initial=self.num_epochs * len(self.train_data_loader),
+            lr_min=2e-7,
+            warmup_lr_init=1e-7,
+            warmup_prefix=True,
+            warmup_t=3,
+            cycle_limit=1,
+            t_in_epochs=False,
+        )
+        progress_bar = tqdm(range(self.num_epochs))
+        for epoch in progress_bar:
+            running_loss = 0.0
+            for batch_idx, batch in enumerate(self.train_data_loader):
+                self.optimizer.zero_grad()
+                processed_input = self.processor(text=batch["caption"],
+                                                 images=batch["image"],
+                                                 return_tensors="pt",
+                                                 padding=True,
+                                                 max_length=128,
+                                                 truncation=True
+                )
+                outputs = self.model(input_ids=processed_input['input_ids'].squeeze().to(self.device),
+                                     pixel_values=processed_input['pixel_values'].squeeze().to(self.device),
+                                     attention_mask=processed_input['attention_mask'].squeeze().to(self.device),
+                                     return_loss=True
+                )
+                loss = outputs.loss
+                loss.backward()
+                running_loss += loss.item() * len(batch["caption"])
+                self.optimizer.step()
+                lr_scheduler.step_update(batch_idx + epoch * len(self.train_data_loader))
+
+            print(f"Epoch {epoch+1}/{self.num_epochs} Loss: {running_loss/len(self.train_data_loader.dataset):.4f}")
+            progress_bar.set_postfix(
+                epoch="{}/{}".format(epoch+1,self.num_epochs),
+                loss=running_loss/len(self.train_data_loader.dataset),
+                lr=self.optimizer.param_groups[0]["lr"]
+            )
+
+        if self.save_model:
+            if self.data_name not in ['MS_COCO','all']:
+                torch.save(self.model.state_dict(), self.save_model_path + f'clip_model_dataset_{self.data_name}_method_{self.method}_num_epochs_{self.num_epochs}_data_seed_{self.data_seed}.pt')
+                print(f"Saving fine-tuned model as clip_model_dataset_{self.data_name}_method_{self.method}_num_epochs_{self.num_epochs}_data_seed_{self.data_seed}.pt")
+            else:
+                torch.save(self.model.state_dict(), self.save_model_path + f'clip_model_dataset_{self.data_name}_method_{self.method}_num_epochs_{self.num_epochs}.pt')
+                print(f"Saving fine-tuned model as clip_model_dataset_{self.data_name}_method_{self.method}_num_epochs_{self.num_epochs}.pt")
+
+    def eval(self):
+        self.model.eval()
+        nb_batches = len(self.val_data_loader)
+        tqdm_object = tqdm(self.val_data_loader, total=len(self.val_data_loader))
+        epoch_loss = 0.0
+        for i, batch in enumerate(tqdm_object):
+            processed_input = self.processor(text=batch["caption"],
+                                                 images=batch["image"],
+                                                 return_tensors="pt",
+                                                 padding=True,
+                                                 max_length=128,
+                                                 truncation=True
+                )
+            outputs = self.model(
+                input_ids=processed_input['input_ids'].squeeze().to(self.device),
+                attention_mask=processed_input['attention_mask'].squeeze().to(self.device),
+                pixel_values=processed_input['pixel_values'].squeeze().to(self.device),
+                return_loss=True)
+            loss, logits_per_image = outputs.loss, outputs.logits_per_image
+            epoch_loss += loss.item()
+            tqdm_object.set_postfix(
+                batch="{}/{}".format(i+1,nb_batches),
+                dev_loss=loss.item(),
+                )
+        epoch_loss = epoch_loss / nb_batches
+        print(f"Eval loss: {epoch_loss}")
+
+def main():
+    import os
+    #os.environ['HF_HOME'] = ''    Add path for saved hugging face models
+
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--num_epochs', type=int, default=20)
+    parser.add_argument('--data_name', type=str, default="MS_COCO", choices=["MS_COCO","base","medium","all"])
+    parser.add_argument('--learning_rate', type=float, default=1e-5)
+    parser.add_argument('--batch_size', type=int, default=32)
+    parser.add_argument('--save_model', action='store_true', default=False)
+    parser.add_argument('--method', type=str, choices=['COCO_CF','APGD_1','APGD_4','NONE'])
+    parser.add_argument('--save_model_path', type=str, default="./fine_tuned_clip_models")
+    parser.add_argument(
+        "--data_seeds",
+        nargs="+",
+        type=int,
+        default=[107],
+        help="Seeds to use for each trial for picking demonstrations and eval sets",
+    )
+    args = parser.parse_args()
+    if args.data_name == 'MS_COCO':
+        assert args.data_name == 'MS_COCO' and args.method == 'NONE', "Only NONE method is allowed with MS_COCO dataset"
+
+    from torch.utils.data import DataLoader
+    from coco_cf_loader import MS_COCO_dataset, custom_collate_fn
+
+    torch.manual_seed(42)
+
+    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+    model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32").to(device)
+    processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
+
+
+    for data_seed in args.data_seeds:
+
+        if args.data_name not in ['MS_COCO', 'all']:
+            print(f"Data Seed: {data_seed} | Data Name: {args.data_name} | Method: {args.method}")
+            dataset = MS_COCO_dataset(base_dir=f'./clip_train_datasets/MS_COCO_{args.method}',
+                                      annotation_file=f'/json_files/data_name_{args.data_name}_data_seed_{data_seed}.json')
+        elif args.data_name == 'all':
+            print(f"Data Name: {args.data_name} | Method: {args.method}")
+            dataset = MS_COCO_dataset(base_dir=f'./clip_train_datasets/MS_COCO_{args.method}',
+                                      annotation_file=f'/json_files/data_name_{args.data_name}.json')
+        else:
+            print(f"Data Name: {args.data_name} | Method: {args.method}")
+            dataset = MS_COCO_dataset(base_dir=f'./clip_train_datasets/MS_COCO',
+                                      annotation_file=f'/ms_coco_captions.json')
+
+        train_size = int(0.8 * len(dataset))  # 80% for training
+        val_size = len(dataset) - train_size   # 20% for validation
+
+        # Randomly split into training and validation datasets
+        train_dataset, val_dataset = torch.utils.data.random_split(dataset, [train_size, val_size])
+
+        # Optional: Create DataLoaders for each subset
+        train_loader = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, collate_fn=custom_collate_fn)
+        val_loader = DataLoader(val_dataset, batch_size=128, shuffle=False, collate_fn=custom_collate_fn,drop_last=True)
+
+        trainer = ModelTrainer(model=model,
+                            processor=processor,
+                            data_name=args.data_name,
+                            train_data_loader=train_loader,
+                            val_data_loader=val_loader,
+                            num_epochs=args.num_epochs,
+                            learning_rate=args.learning_rate,
+                            weight_decay=1e-3,
+                            device=device,
+                            data_seed=data_seed,
+                            save_model=args.save_model,
+                            save_model_path=args.save_model_path,
+                            method=args.method,
+        )
+
+        trainer.train()
+        trainer.eval()
+        if args.data_name in ['MS_COCO','all']:
+            break
+
+
+if __name__ == "__main__":
+    main()

vlm_eval/coco_cf_loader.py

@@ -0,0 +1,90 @@
+from torch.utils.data import Dataset
+from torch.utils.data import DataLoader
+import os
+import json
+from PIL import Image
+
+
+class MS_COCO_dataset(Dataset):
+
+    def __init__(self, base_dir, annotation_file=None):
+
+        self.data= []
+        self.img_dir = base_dir + '/images'
+        self.annotation_file = base_dir + annotation_file
+
+        with open(self.annotation_file, 'r') as file:
+            for line in file:
+                self.data.append(json.loads(line))
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        # Extract the relevant info from the JSONL entry
+        img_name = os.path.join(self.img_dir, f"{self.data[idx]['image_name']}")
+        caption = self.data[idx]['caption']
+        sample_id = self.data[idx]['image_id']
+
+        # Load the image using PIL
+        img = Image.open(img_name)
+
+        return {"id": sample_id, 
+                "image": img,
+                "caption": caption
+                }
+
+class COCO_CF_dataset(Dataset):
+
+    def __init__(self, base_dir):
+
+        self.data= []
+        self.img_dir = base_dir + '/images'
+        self.annotation_file = base_dir + "/examples.jsonl"
+
+        with open(self.annotation_file, 'r') as file:
+            for line in file:
+                self.data.append(json.loads(line))
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        # Extract the relevant info from the JSONL entry
+        img_0_name = os.path.join(self.img_dir, f"{self.data[idx]['image_0']}.jpg")
+        img_1_name = os.path.join(self.img_dir, f"{self.data[idx]['image_1']}.jpg")
+
+        caption_0 = self.data[idx]['caption_0']
+        caption_1 = self.data[idx]['caption_1']
+        sample_id = self.data[idx]['id']
+
+        # Load the image using PIL
+        img_0 = Image.open(img_0_name)
+        img_1 = Image.open(img_1_name)
+
+        return {"id": sample_id, 
+                "caption_0": caption_0, 
+                "caption_1": caption_1, 
+                "image_0": img_0, 
+                "image_1": img_1}
+
+def custom_collate_fn(batch):
+    collated_batch = {}
+    for key in batch[0].keys():
+        collated_batch[key] = [item[key] for item in batch]
+    return collated_batch
+
+if __name__ == "__main__":
+
+    base_dir = '/home/htc/kchitranshi/SCRATCH/MS_COCO/'   
+    data = MS_COCO_dataset(base_dir=base_dir)
+    data_loader = DataLoader(data, batch_size=10,collate_fn=custom_collate_fn)
+    
+    for batch in data_loader:
+        print(batch)
+        break
+
+
+
+
+    

vlm_eval/create_clip_dataset.py

@@ -0,0 +1,65 @@
+import json
+import torch
+import numpy as np
+import random
+
+
+
+def main():
+
+    # Intialising seeds for data
+    data_seeds = [i for i in range(107,122)]
+
+    ms_coco_base_anno_path = "./clip_train_datasets/MS_COCO/ms_coco_captions.json"
+    attack_base_anno_path = "./clip_train_datasets/COCO_CF/examples.jsonl"
+
+    data_names = ["base","medium","all"]
+
+    ms_coco_array = []
+    attack_array = []
+
+    with open(ms_coco_base_anno_path, 'r') as file:
+        for line in file:
+            ms_coco_array.append(json.loads(line))
+
+
+    with open(attack_base_anno_path, 'r') as file:
+        for line in file:
+            attack_array.append(json.loads(line))
+
+    for data_name in data_names:
+        for data_seed in data_seeds:
+            if data_name == "base":
+                num_ms_coco_samples = 8705
+                num_attacks_samples = 4353 # These many pairs of samples with their counterfactuals or adv attack samples. Effectively 8706 in total.
+            elif data_name == "medium":
+                num_ms_coco_samples = 17410
+                num_attacks_samples = int(0.75 * 17410) # These many pairs of samples with their counterfactuals or adv attack samples. Effectively 26115 in total.
+            elif data_name == "all":
+                num_ms_coco_samples = 17410
+                num_attacks_samples = 17410 # These many pairs of samples with their counterfactuals or adv attack samples. Effectively 34820 in total.
+
+            np.random.seed(data_seed)
+            ms_coco_rand_indices = np.random.choice(len(ms_coco_array), num_ms_coco_samples, replace=False)
+            attack_rand_indices = np.random.choice(len(attack_array), num_attacks_samples, replace=False)
+
+            ms_coco_samples = [ms_coco_array[int(i)] for i in ms_coco_rand_indices]
+            attack_samples = [attack_array[int(i)] for i in attack_rand_indices]
+            attack_samples = [{"image_id": batch["id"], "image_name": batch[f"image_{i}"] + ".jpg", "caption": batch[f"caption_{i}"]} for batch in attack_samples for i in range(2)]
+
+            random.seed(42)
+            combined_dataset = ms_coco_samples + attack_samples
+
+            random.shuffle(combined_dataset)
+
+            if data_name != 'all':
+                with open(f"./clip_train_datasets/MS_COCO_APGD_4/json_files/data_name_{data_name}_data_seed_{data_seed}.json", 'w') as file:
+                    for line in combined_dataset:
+                        file.write(json.dumps(line) + '\n')
+            else:
+                with open(f"./clip_train_datasets/MS_COCO_APGD_4/json_files/data_name_{data_name}.json", 'w') as file:
+                    for line in combined_dataset:
+                        file.write(json.dumps(line) + '\n')
+
+if __name__ == "__main__":
+    main()

vlm_eval/datasets_classes_templates.py

@@ -0,0 +1,822 @@
+# Code taken and adapted from https://github.com/openai/CLIP/blob/main/data/prompts.md
+
+CIFAR10_CLASSES_TEMPLATES = {
+    'classes' : [
+        'airplane',
+        'automobile',
+        'bird',
+        'cat',
+        'deer',
+        'dog',
+        'frog',
+        'horse',
+        'ship',
+        'truck',
+    ],
+
+    'templates' : [
+        'a photo of a {}.',
+        'a blurry photo of a {}.',
+        'a black and white photo of a {}.',
+        'a low contrast photo of a {}.',
+        'a high contrast photo of a {}.',
+        'a bad photo of a {}.',
+        'a good photo of a {}.',
+        'a photo of a small {}.',
+        'a photo of a big {}.',
+        'a photo of the {}.',
+        'a blurry photo of the {}.',
+        'a black and white photo of the {}.',
+        'a low contrast photo of the {}.',
+        'a high contrast photo of the {}.',
+        'a bad photo of the {}.',
+        'a good photo of the {}.',
+        'a photo of the small {}.',
+        'a photo of the big {}.',
+    ]
+}
+
+CIFAR100_CLASSES_TEMPLATES = {
+    'classes' : [
+        'apple',
+        'aquarium fish',
+        'baby',
+        'bear',
+        'beaver',
+        'bed',
+        'bee',
+        'beetle',
+        'bicycle',
+        'bottle',
+        'bowl',
+        'boy',
+        'bridge',
+        'bus',
+        'butterfly',
+        'camel',
+        'can',
+        'castle',
+        'caterpillar',
+        'cattle',
+        'chair',
+        'chimpanzee',
+        'clock',
+        'cloud',
+        'cockroach',
+        'couch',
+        'crab',
+        'crocodile',
+        'cup',
+        'dinosaur',
+        'dolphin',
+        'elephant',
+        'flatfish',
+        'forest',
+        'fox',
+        'girl',
+        'hamster',
+        'house',
+        'kangaroo',
+        'keyboard',
+        'lamp',
+        'lawn mower',
+        'leopard',
+        'lion',
+        'lizard',
+        'lobster',
+        'man',
+        'maple tree',
+        'motorcycle',
+        'mountain',
+        'mouse',
+        'mushroom',
+        'oak tree',
+        'orange',
+        'orchid',
+        'otter',
+        'palm tree',
+        'pear',
+        'pickup truck',
+        'pine tree',
+        'plain',
+        'plate',
+        'poppy',
+        'porcupine',
+        'possum',
+        'rabbit',
+        'raccoon',
+        'ray',
+        'road',
+        'rocket',
+        'rose',
+        'sea',
+        'seal',
+        'shark',
+        'shrew',
+        'skunk',
+        'skyscraper',
+        'snail',
+        'snake',
+        'spider',
+        'squirrel',
+        'streetcar',
+        'sunflower',
+        'sweet pepper',
+        'table',
+        'tank',
+        'telephone',
+        'television',
+        'tiger',
+        'tractor',
+        'train',
+        'trout',
+        'tulip',
+        'turtle',
+        'wardrobe',
+        'whale',
+        'willow tree',
+        'wolf',
+        'woman',
+        'worm',
+    ],
+
+    'templates' : [
+        'a photo of a {}.',
+        'a blurry photo of a {}.',
+        'a black and white photo of a {}.',
+        'a low contrast photo of a {}.',
+        'a high contrast photo of a {}.',
+        'a bad photo of a {}.',
+        'a good photo of a {}.',
+        'a photo of a small {}.',
+        'a photo of a big {}.',
+        'a photo of the {}.',
+        'a blurry photo of the {}.',
+        'a black and white photo of the {}.',
+        'a low contrast photo of the {}.',
+        'a high contrast photo of the {}.',
+        'a bad photo of the {}.',
+        'a good photo of the {}.',
+        'a photo of the small {}.',
+        'a photo of the big {}.',
+    ]
+}
+
+ImageNet_CLASSES_TEMPLATES = {
+    'classes' : ["tench", "goldfish", "great white shark", "tiger shark", "hammerhead shark", "electric ray", "stingray", "rooster", "hen", "ostrich", "brambling", "goldfinch", "house finch", "junco", "indigo bunting", "American robin", "bulbul", "jay", "magpie", "chickadee", "American dipper", "kite (bird of prey)", "bald eagle", "vulture", "great grey owl", "fire salamander", "smooth newt", "newt", "spotted salamander", "axolotl", "American bullfrog", "tree frog", "tailed frog", "loggerhead sea turtle", "leatherback sea turtle", "mud turtle", "terrapin", "box turtle", "banded gecko", "green iguana", "Carolina anole", "desert grassland whiptail lizard", "agama", "frilled-necked lizard", "alligator lizard", "Gila monster", "European green lizard", "chameleon", "Komodo dragon", "Nile crocodile", "American alligator", "triceratops", "worm snake", "ring-necked snake", "eastern hog-nosed snake", "smooth green snake", "kingsnake", "garter snake", "water snake", "vine snake", "night snake", "boa constrictor", "African rock python", "Indian cobra", "green mamba", "sea snake", "Saharan horned viper", "eastern diamondback rattlesnake", "sidewinder rattlesnake", "trilobite", "harvestman", "scorpion", "yellow garden spider", "barn spider", "European garden spider", "southern black widow", "tarantula", "wolf spider", "tick", "centipede", "black grouse", "ptarmigan", "ruffed grouse", "prairie grouse", "peafowl", "quail", "partridge", "african grey parrot", "macaw", "sulphur-crested cockatoo", "lorikeet", "coucal", "bee eater", "hornbill", "hummingbird", "jacamar", "toucan", "duck", "red-breasted merganser", "goose", "black swan", "tusker", "echidna", "platypus", "wallaby", "koala", "wombat", "jellyfish", "sea anemone", "brain coral", "flatworm", "nematode", "conch", "snail", "slug", "sea slug", "chiton", "chambered nautilus", "Dungeness crab", "rock crab", "fiddler crab", "red king crab", "American lobster", "spiny lobster", "crayfish", "hermit crab", "isopod", "white stork", "black stork", "spoonbill", "flamingo", "little blue heron", "great egret", "bittern bird", "crane bird", "limpkin", "common gallinule", "American coot", "bustard", "ruddy turnstone", "dunlin", "common redshank", "dowitcher", "oystercatcher", "pelican", "king penguin", "albatross", "grey whale", "killer whale", "dugong", "sea lion", "Chihuahua", "Japanese Chin", "Maltese", "Pekingese", "Shih Tzu", "King Charles Spaniel", "Papillon", "toy terrier", "Rhodesian Ridgeback", "Afghan Hound", "Basset Hound", "Beagle", "Bloodhound", "Bluetick Coonhound", "Black and Tan Coonhound", "Treeing Walker Coonhound", "English foxhound", "Redbone Coonhound", "borzoi", "Irish Wolfhound", "Italian Greyhound", "Whippet", "Ibizan Hound", "Norwegian Elkhound", "Otterhound", "Saluki", "Scottish Deerhound", "Weimaraner", "Staffordshire Bull Terrier", "American Staffordshire Terrier", "Bedlington Terrier", "Border Terrier", "Kerry Blue Terrier", "Irish Terrier", "Norfolk Terrier", "Norwich Terrier", "Yorkshire Terrier", "Wire Fox Terrier", "Lakeland Terrier", "Sealyham Terrier", "Airedale Terrier", "Cairn Terrier", "Australian Terrier", "Dandie Dinmont Terrier", "Boston Terrier", "Miniature Schnauzer", "Giant Schnauzer", "Standard Schnauzer", "Scottish Terrier", "Tibetan Terrier", "Australian Silky Terrier", "Soft-coated Wheaten Terrier", "West Highland White Terrier", "Lhasa Apso", "Flat-Coated Retriever", "Curly-coated Retriever", "Golden Retriever", "Labrador Retriever", "Chesapeake Bay Retriever", "German Shorthaired Pointer", "Vizsla", "English Setter", "Irish Setter", "Gordon Setter", "Brittany dog", "Clumber Spaniel", "English Springer Spaniel", "Welsh Springer Spaniel", "Cocker Spaniel", "Sussex Spaniel", "Irish Water Spaniel", "Kuvasz", "Schipperke", "Groenendael dog", "Malinois", "Briard", "Australian Kelpie", "Komondor", "Old English Sheepdog", "Shetland Sheepdog", "collie", "Border Collie", "Bouvier des Flandres dog", "Rottweiler", "German Shepherd Dog", "Dobermann", "Miniature Pinscher", "Greater Swiss Mountain Dog", "Bernese Mountain Dog", "Appenzeller Sennenhund", "Entlebucher Sennenhund", "Boxer", "Bullmastiff", "Tibetan Mastiff", "French Bulldog", "Great Dane", "St. Bernard", "husky", "Alaskan Malamute", "Siberian Husky", "Dalmatian", "Affenpinscher", "Basenji", "pug", "Leonberger", "Newfoundland dog", "Great Pyrenees dog", "Samoyed", "Pomeranian", "Chow Chow", "Keeshond", "brussels griffon", "Pembroke Welsh Corgi", "Cardigan Welsh Corgi", "Toy Poodle", "Miniature Poodle", "Standard Poodle", "Mexican hairless dog (xoloitzcuintli)", "grey wolf", "Alaskan tundra wolf", "red wolf or maned wolf", "coyote", "dingo", "dhole", "African wild dog", "hyena", "red fox", "kit fox", "Arctic fox", "grey fox", "tabby cat", "tiger cat", "Persian cat", "Siamese cat", "Egyptian Mau", "cougar", "lynx", "leopard", "snow leopard", "jaguar", "lion", "tiger", "cheetah", "brown bear", "American black bear", "polar bear", "sloth bear", "mongoose", "meerkat", "tiger beetle", "ladybug", "ground beetle", "longhorn beetle", "leaf beetle", "dung beetle", "rhinoceros beetle", "weevil", "fly", "bee", "ant", "grasshopper", "cricket insect", "stick insect", "cockroach", "praying mantis", "cicada", "leafhopper", "lacewing", "dragonfly", "damselfly", "red admiral butterfly", "ringlet butterfly", "monarch butterfly", "small white butterfly", "sulphur butterfly", "gossamer-winged butterfly", "starfish", "sea urchin", "sea cucumber", "cottontail rabbit", "hare", "Angora rabbit", "hamster", "porcupine", "fox squirrel", "marmot", "beaver", "guinea pig", "common sorrel horse", "zebra", "pig", "wild boar", "warthog", "hippopotamus", "ox", "water buffalo", "bison", "ram (adult male sheep)", "bighorn sheep", "Alpine ibex", "hartebeest", "impala (antelope)", "gazelle", "arabian camel", "llama", "weasel", "mink", "European polecat", "black-footed ferret", "otter", "skunk", "badger", "armadillo", "three-toed sloth", "orangutan", "gorilla", "chimpanzee", "gibbon", "siamang", "guenon", "patas monkey", "baboon", "macaque", "langur", "black-and-white colobus", "proboscis monkey", "marmoset", "white-headed capuchin", "howler monkey", "titi monkey", "Geoffroy's spider monkey", "common squirrel monkey", "ring-tailed lemur", "indri", "Asian elephant", "African bush elephant", "red panda", "giant panda", "snoek fish", "eel", "silver salmon", "rock beauty fish", "clownfish", "sturgeon", "gar fish", "lionfish", "pufferfish", "abacus", "abaya", "academic gown", "accordion", "acoustic guitar", "aircraft carrier", "airliner", "airship", "altar", "ambulance", "amphibious vehicle", "analog clock", "apiary", "apron", "trash can", "assault rifle", "backpack", "bakery", "balance beam", "balloon", "ballpoint pen", "Band-Aid", "banjo", "baluster / handrail", "barbell", "barber chair", "barbershop", "barn", "barometer", "barrel", "wheelbarrow", "baseball", "basketball", "bassinet", "bassoon", "swimming cap", "bath towel", "bathtub", "station wagon", "lighthouse", "beaker", "military hat (bearskin or shako)", "beer bottle", "beer glass", "bell tower", "baby bib", "tandem bicycle", "bikini", "ring binder", "binoculars", "birdhouse", "boathouse", "bobsleigh", "bolo tie", "poke bonnet", "bookcase", "bookstore", "bottle cap", "hunting bow", "bow tie", "brass memorial plaque", "bra", "breakwater", "breastplate", "broom", "bucket", "buckle", "bulletproof vest", "high-speed train", "butcher shop", "taxicab", "cauldron", "candle", "cannon", "canoe", "can opener", "cardigan", "car mirror", "carousel", "tool kit", "cardboard box / carton", "car wheel", "automated teller machine", "cassette", "cassette player", "castle", "catamaran", "CD player", "cello", "mobile phone", "chain", "chain-link fence", "chain mail", "chainsaw", "storage chest", "chiffonier", "bell or wind chime", "china cabinet", "Christmas stocking", "church", "movie theater", "cleaver", "cliff dwelling", "cloak", "clogs", "cocktail shaker", "coffee mug", "coffeemaker", "spiral or coil", "combination lock", "computer keyboard", "candy store", "container ship", "convertible", "corkscrew", "cornet", "cowboy boot", "cowboy hat", "cradle", "construction crane", "crash helmet", "crate", "infant bed", "Crock Pot", "croquet ball", "crutch", "cuirass", "dam", "desk", "desktop computer", "rotary dial telephone", "diaper", "digital clock", "digital watch", "dining table", "dishcloth", "dishwasher", "disc brake", "dock", "dog sled", "dome", "doormat", "drilling rig", "drum", "drumstick", "dumbbell", "Dutch oven", "electric fan", "electric guitar", "electric locomotive", "entertainment center", "envelope", "espresso machine", "face powder", "feather boa", "filing cabinet", "fireboat", "fire truck", "fire screen", "flagpole", "flute", "folding chair", "football helmet", "forklift", "fountain", "fountain pen", "four-poster bed", "freight car", "French horn", "frying pan", "fur coat", "garbage truck", "gas mask or respirator", "gas pump", "goblet", "go-kart", "golf ball", "golf cart", "gondola", "gong", "gown", "grand piano", "greenhouse", "radiator grille", "grocery store", "guillotine", "hair clip", "hair spray", "half-track", "hammer", "hamper", "hair dryer", "hand-held computer", "handkerchief", "hard disk drive", "harmonica", "harp", "combine harvester", "hatchet", "holster", "home theater", "honeycomb", "hook", "hoop skirt", "gymnastic horizontal bar", "horse-drawn vehicle", "hourglass", "iPod", "clothes iron", "carved pumpkin", "jeans", "jeep", "T-shirt", "jigsaw puzzle", "rickshaw", "joystick", "kimono", "knee pad", "knot", "lab coat", "ladle", "lampshade", "laptop computer", "lawn mower", "lens cap", "letter opener", "library", "lifeboat", "lighter", "limousine", "ocean liner", "lipstick", "slip-on shoe", "lotion", "music speaker", "loupe magnifying glass", "sawmill", "magnetic compass", "messenger bag", "mailbox", "tights", 
+    "one-piece bathing suit", "manhole cover", "maraca", "marimba", "mask", "matchstick", "maypole", "maze", "measuring cup", "medicine cabinet", "megalith", "microphone", "microwave oven", 
+    "military uniform", "milk can", "minibus", "miniskirt", "minivan", "missile", "mitten", "mixing bowl", "mobile home", "ford model t", "modem", "monastery", "monitor", "moped", 
+    "mortar and pestle", "graduation cap", "mosque", "mosquito net", "vespa", "mountain bike", "tent", "computer mouse", "mousetrap", "moving van", "muzzle", "metal nail", "neck brace", 
+    "necklace", "baby pacifier", "notebook computer", "obelisk", "oboe", "ocarina", "odometer", "oil filter", "pipe organ", "oscilloscope", "overskirt", "bullock cart", "oxygen mask", 
+    "product packet / packaging", "paddle", "paddle wheel", "padlock", "paintbrush", "pajamas", "palace", "pan flute", "paper towel", "parachute", "parallel bars", "park bench", "parking meter",
+    "railroad car", "patio", "payphone", "pedestal", "pencil case", "pencil sharpener", "perfume", "Petri dish", "photocopier", "plectrum", "Pickelhaube", "picket fence", "pickup truck", "pier",
+    "piggy bank", "pill bottle", "pillow", "ping-pong ball", "pinwheel", "pirate ship", "drink pitcher", "block plane", "planetarium", "plastic bag", "plate rack", "farm plow", "plunger", 
+    "Polaroid camera", "pole", "police van", "poncho", "pool table", "soda bottle", "plant pot", "potter's wheel", "power drill", "prayer rug", "printer", "prison", "missile", "projector", 
+    "hockey puck", "punching bag", "purse", "quill", "quilt", "race car", "racket", "radiator", "radio", "radio telescope", "rain barrel", "recreational vehicle", 
+    "fishing casting reel", "reflex camera", "refrigerator", "remote control", "restaurant", "revolver", "rifle", "rocking chair", "rotisserie", "eraser", "rugby ball", "ruler measuring stick", 
+    "sneaker", "safe", "safety pin", "salt shaker", "sandal", "sarong", "saxophone", "scabbard", "weighing scale", "school bus", "schooner", "scoreboard", "CRT monitor", "screw", "screwdriver", 
+    "seat belt", "sewing machine", "shield", "shoe store", "shoji screen / room divider", "shopping basket", "shopping cart", "shovel", "shower cap", "shower curtain", "ski", 
+    "balaclava ski mask", "sleeping bag", "slide rule", "sliding door", "slot machine", "snorkel", "snowmobile", "snowplow", "soap dispenser", "soccer ball", "sock", 
+    "solar thermal collector", "sombrero", "soup bowl", "keyboard space bar", "space heater", "space shuttle", "spatula", "motorboat", "spider web", "spindle", "sports car", 
+    "spotlight", "stage", "steam locomotive", "through arch bridge", "steel drum", "stethoscope", "scarf", "stone wall", "stopwatch", "stove", "strainer", "tram", "stretcher", "couch", "stupa", "submarine", "suit", "sundial", "sunglasses", "sunglasses", "sunscreen", "suspension bridge", "mop", "sweatshirt", "swim trunks / shorts", "swing",
+     "electrical switch", "syringe", "table lamp", "tank", "tape player", "teapot", "teddy bear", "television", "tennis ball", "thatched roof", "front curtain", "thimble", "threshing machine", "throne", "tile roof", "toaster", "tobacco shop", "toilet seat", "torch", "totem pole", "tow truck", "toy store", "tractor", "semi-trailer truck", "tray", "trench coat", "tricycle", "trimaran", "tripod", "triumphal arch", "trolleybus", "trombone", "hot tub", "turnstile", "typewriter keyboard", "umbrella", "unicycle", "upright piano", "vacuum cleaner", "vase", "vaulted or arched ceiling", "velvet fabric", "vending machine", "vestment", "viaduct", "violin", "volleyball", "waffle iron", "wall clock", "wallet", "wardrobe", "military aircraft", "sink", "washing machine", "water bottle", "water jug", "water tower", "whiskey jug", "whistle", "hair wig", "window screen", "window shade", "Windsor tie", "wine bottle", "airplane wing", "wok", "wooden spoon", "wool", "split-rail fence", "shipwreck", "sailboat", "yurt", "website", "comic book", "crossword", "traffic or street sign", "traffic light", "dust jacket", "menu", "plate", "guacamole", "consomme", "hot pot", "trifle", "ice cream", "popsicle", "baguette", "bagel", "pretzel", "cheeseburger", "hot dog", "mashed potatoes", "cabbage", "broccoli", "cauliflower", "zucchini", "spaghetti squash", "acorn squash", "butternut squash", "cucumber", "artichoke", "bell pepper", "cardoon", "mushroom", "Granny Smith apple", "strawberry", "orange", "lemon", "fig", "pineapple", "banana", "jackfruit", "cherimoya (custard apple)", "pomegranate", "hay", "carbonara", "chocolate syrup", "dough", "meatloaf", "pizza", "pot pie", "burrito", "red wine", "espresso", "tea cup", "eggnog", "mountain", "bubble", "cliff", "coral reef", "geyser", "lakeshore", "promontory", "sandbar", "beach", "valley", "volcano", "baseball player", "bridegroom", "scuba diver", "rapeseed", "daisy", "yellow lady's slipper", "corn", "acorn", "rose hip", "horse chestnut seed", "coral fungus", "agaric", "gyromitra", "stinkhorn mushroom", "earth star fungus", "hen of the woods mushroom", "bolete", "corn cob", "toilet paper"]
+    ,
+    'templates' : [
+        'a bad photo of a {}.',
+        'a photo of many {}.',
+        'a sculpture of a {}.',
+        'a photo of the hard to see {}.',
+        'a low resolution photo of the {}.',
+        'a rendering of a {}.',
+        'graffiti of a {}.',
+        'a bad photo of the {}.',
+        'a cropped photo of the {}.',
+        'a tattoo of a {}.',
+        'the embroidered {}.',
+        'a photo of a hard to see {}.',
+        'a bright photo of a {}.',
+        'a photo of a clean {}.',
+        'a photo of a dirty {}.',
+        'a dark photo of the {}.',
+        'a drawing of a {}.',
+        'a photo of my {}.',
+        'the plastic {}.',
+        'a photo of the cool {}.',
+        'a close-up photo of a {}.',
+        'a black and white photo of the {}.',
+        'a painting of the {}.',
+        'a painting of a {}.',
+        'a pixelated photo of the {}.',
+        'a sculpture of the {}.',
+        'a bright photo of the {}.',
+        'a cropped photo of a {}.',
+        'a plastic {}.',
+        'a photo of the dirty {}.',
+        'a jpeg corrupted photo of a {}.',
+        'a blurry photo of the {}.',
+        'a photo of the {}.',
+        'a good photo of the {}.',
+        'a rendering of the {}.',
+        'a {} in a video game.',
+        'a photo of one {}.',
+        'a doodle of a {}.',
+        'a close-up photo of the {}.',
+        'a photo of a {}.',
+        'the origami {}.',
+        'the {} in a video game.',
+        'a sketch of a {}.',
+        'a doodle of the {}.',
+        'a origami {}.',
+        'a low resolution photo of a {}.',
+        'the toy {}.',
+        'a rendition of the {}.',
+        'a photo of the clean {}.',
+        'a photo of a large {}.',
+        'a rendition of a {}.',
+        'a photo of a nice {}.',
+        'a photo of a weird {}.',
+        'a blurry photo of a {}.',
+        'a cartoon {}.',
+        'art of a {}.',
+        'a sketch of the {}.',
+        'a embroidered {}.',
+        'a pixelated photo of a {}.',
+        'itap of the {}.',
+        'a jpeg corrupted photo of the {}.',
+        'a good photo of a {}.',
+        'a plushie {}.',
+        'a photo of the nice {}.',
+        'a photo of the small {}.',
+        'a photo of the weird {}.',
+        'the cartoon {}.',
+        'art of the {}.',
+        'a drawing of the {}.',
+        'a photo of the large {}.',
+        'a black and white photo of a {}.',
+        'the plushie {}.',
+        'a dark photo of a {}.',
+        'itap of a {}.',
+        'graffiti of the {}.',
+        'a toy {}.',
+        'itap of my {}.',
+        'a photo of a cool {}.',
+        'a photo of a small {}.',
+        'a tattoo of the {}.',
+    ]
+}
+
+Caltech101_CLASSES_TEMPLATES = {
+    
+
+    'classes' : ['Faces',
+        'Faces_easy',
+        'Leopards',
+        'Motorbikes',
+        'accordion',
+        'airplanes',
+        'anchor',
+        'ant',
+        'barrel',
+        'bass',
+        'beaver',
+        'binocular',
+        'bonsai',
+        'brain',
+        'brontosaurus',
+        'buddha',
+        'butterfly',
+        'camera',
+        'cannon',
+        'car_side',
+        'ceiling_fan',
+        'cellphone',
+        'chair',
+        'chandelier',
+        'cougar_body',
+        'cougar_face',
+        'crab',
+        'crayfish',
+        'crocodile',
+        'crocodile_head',
+        'cup',
+        'dalmatian',
+        'dollar_bill',
+        'dolphin',
+        'dragonfly',
+        'electric_guitar',
+        'elephant',
+        'emu',
+        'euphonium',
+        'ewer',
+        'ferry',
+        'flamingo',
+        'flamingo_head',
+        'garfield',
+        'gerenuk',
+        'gramophone',
+        'grand_piano',
+        'hawksbill',
+        'headphone',
+        'hedgehog',
+        'helicopter',
+        'ibis',
+        'inline_skate',
+        'joshua_tree',
+        'kangaroo',
+        'ketch',
+        'lamp',
+        'laptop',
+        'llama',
+        'lobster',
+        'lotus',
+        'mandolin',
+        'mayfly',
+        'menorah',
+        'metronome',
+        'minaret',
+        'nautilus',
+        'octopus',
+        'okapi',
+        'pagoda',
+        'panda',
+        'pigeon',
+        'pizza',
+        'platypus',
+        'pyramid',
+        'revolver',
+        'rhino',
+        'rooster',
+        'saxophone',
+        'schooner',
+        'scissors',
+        'scorpion',
+        'sea_horse',
+        'snoopy',
+        'soccer_ball',
+        'stapler',
+        'starfish',
+        'stegosaurus',
+        'stop_sign',
+        'strawberry',
+        'sunflower',
+        'tick',
+        'trilobite',
+        'umbrella',
+        'watch',
+        'water_lilly',
+        'wheelchair',
+        'wild_cat',
+        'windsor_chair',
+        'wrench',
+        'yin_yang']
+        ,
+
+
+    'templates' : [
+        'a photo of a {}.',
+        'a painting of a {}.',
+        'a plastic {}.',
+        'a sculpture of a {}.',
+        'a sketch of a {}.',
+        'a tattoo of a {}.',
+        'a toy {}.',
+        'a rendition of a {}.',
+        'a embroidered {}.',
+        'a cartoon {}.',
+        'a {} in a video game.',
+        'a plushie {}.',
+        'a origami {}.',
+        'art of a {}.',
+        'graffiti of a {}.',
+        'a drawing of a {}.',
+        'a doodle of a {}.',
+        'a photo of the {}.',
+        'a painting of the {}.',
+        'the plastic {}.',
+        'a sculpture of the {}.',
+        'a sketch of the {}.',
+        'a tattoo of the {}.',
+        'the toy {}.',
+        'a rendition of the {}.',
+        'the embroidered {}.',
+        'the cartoon {}.',
+        'the {} in a video game.',
+        'the plushie {}.',
+        'the origami {}.',
+        'art of the {}.',
+        'graffiti of the {}.',
+        'a drawing of the {}.',
+        'a doodle of the {}.',
+    ]
+}
+
+Caltech256_CLASSES_TEMPLATES = {
+    'classes' : [
+        'ak47',
+        'american flag',
+        'backpack',
+        'baseball bat',
+        'baseball glove',
+        'basketball hoop',
+        'bat',
+        'bathtub',
+        'bear',
+        'beer mug',
+        'billiards',
+        'binoculars',
+        'birdbath',
+        'blimp',
+        'bonsai',
+        'boom box',
+        'bowling ball',
+        'bowling pin',
+        'boxing glove',
+        'brain',
+        'breadmaker',
+        'buddha',
+        'bulldozer',
+        'butterfly',
+        'cactus',
+        'cake',
+        'calculator',
+        'camel',
+        'cannon',
+        'canoe',
+        'car tire',
+        'cartman',
+        'cd',
+        'centipede',
+        'cereal box',
+        'chandelier',
+        'chess board',
+        'chimp',
+        'chopsticks',
+        'cockroach',
+        'coffee mug',
+        'coffin',
+        'coin',
+        'comet',
+        'computer keyboard',
+        'computer monitor',
+        'computer mouse',
+        'conch',
+        'cormorant',
+        'covered wagon',
+        'cowboy hat',
+        'crab',
+        'desk globe',
+        'diamond ring',
+        'dice',
+        'dog',
+        'dolphin',
+        'doorknob',
+        'drinking straw',
+        'duck',
+        'dumb bell',
+        'eiffel tower',
+        'electric guitar',
+        'elephant',
+        'elk',
+        'ewer',
+        'eyeglasses',
+        'fern',
+        'fighter jet',
+        'fire extinguisher',
+        'fire hydrant',
+        'fire truck',
+        'fireworks',
+        'flashlight',
+        'floppy disk',
+        'football helmet',
+        'french horn',
+        'fried egg',
+        'frisbee',
+        'frog',
+        'frying pan',
+        'galaxy',
+        'gas pump',
+        'giraffe',
+        'goat',
+        'golden gate bridge',
+        'goldfish',
+        'golf ball',
+        'goose',
+        'gorilla',
+        'grand piano',
+        'grapes',
+        'grasshopper',
+        'guitar pick',
+        'hamburger',
+        'hammock',
+        'harmonica',
+        'harp',
+        'harpsichord',
+        'hawksbill',
+        'head phones',
+        'helicopter',
+        'hibiscus',
+        'homer simpson',
+        'horse',
+        'horseshoe crab',
+        'hot air balloon',
+        'hot dog',
+        'hot tub',
+        'hourglass',
+        'house fly',
+        'human skeleton',
+        'hummingbird',
+        'ibis',
+        'ice cream cone',
+        'iguana',
+        'ipod',
+        'iris',
+        'jesus christ',
+        'joy stick',
+        'kangaroo',
+        'kayak',
+        'ketch',
+        'killer whale',
+        'knife',
+        'ladder',
+        'laptop',
+        'lathe',
+        'leopards',
+        'license plate',
+        'lightbulb',
+        'light house',
+        'lightning',
+        'llama',
+        'mailbox',
+        'mandolin',
+        'mars',
+        'mattress',
+        'megaphone',
+        'menorah',
+        'microscope',
+        'microwave',
+        'minaret',
+        'minotaur',
+        'motorbikes',
+        'mountain bike',
+        'mushroom',
+        'mussels',
+        'necktie',
+        'octopus',
+        'ostrich',
+        'owl',
+        'palm pilot',
+        'palm tree',
+        'paperclip',
+        'paper shredder',
+        'pci card',
+        'penguin',
+        'people',
+        'pez dispenser',
+        'photocopier',
+        'picnic table',
+        'playing card',
+        'porcupine',
+        'pram',
+        'praying mantis',
+        'pyramid',
+        'raccoon',
+        'radio telescope',
+        'rainbow',
+        'refrigerator',
+        'revolver',
+        'rifle',
+        'rotary phone',
+        'roulette wheel',
+        'saddle',
+        'saturn',
+        'school bus',
+        'scorpion',
+        'screwdriver',
+        'segway',
+        'self propelled lawn mower',
+        'sextant',
+        'sheet music',
+        'skateboard',
+        'skunk',
+        'skyscraper',
+        'smokestack',
+        'snail',
+        'snake',
+        'sneaker',
+        'snowmobile',
+        'soccer ball',
+        'socks',
+        'soda can',
+        'spaghetti',
+        'speed boat',
+        'spider',
+        'spoon',
+        'stained glass',
+        'starfish',
+        'steering wheel',
+        'stirrups',
+        'sunflower',
+        'superman',
+        'sushi',
+        'swan',
+        'swiss army knife',
+        'sword',
+        'syringe',
+        'tambourine',
+        'teapot',
+        'teddy bear',
+        'teepee',
+        'telephone box',
+        'tennis ball',
+        'tennis court',
+        'tennis racket',
+        'theodolite',
+        'toaster',
+        'tomato',
+        'tombstone',
+        'top hat',
+        'touring bike',
+        'tower pisa',
+        'traffic light',
+        'treadmill',
+        'triceratops',
+        'tricycle',
+        'trilobite',
+        'tripod',
+        't shirt',
+        'tuning fork',
+        'tweezer',
+        'umbrella',
+        'unicorn',
+        'vcr',
+        'video projector',
+        'washing machine',
+        'watch',
+        'waterfall',
+        'watermelon',
+        'welding mask',
+        'wheelbarrow',
+        'windmill',
+        'wine bottle',
+        'xylophone',
+        'yarmulke',
+        'yo yo',
+        'zebra',
+        'airplanes',
+        'car side',
+        'faces easy',
+        'greyhound',
+        'tennis shoes',
+        'toad',
+        'clutter'
+    ],
+
+    'templates' : [
+        'a photo of a {}.',
+        'a painting of a {}.',
+        'a plastic {}.',
+        'a sculpture of a {}.',
+        'a sketch of a {}.',
+        'a tattoo of a {}.',
+        'a toy {}.',
+        'a rendition of a {}.',
+        'a embroidered {}.',
+        'a cartoon {}.',
+        'a {} in a video game.',
+        'a plushie {}.',
+        'a origami {}.',
+        'art of a {}.',
+        'graffiti of a {}.',
+        'a drawing of a {}.',
+        'a doodle of a {}.',
+        'a photo of the {}.',
+        'a painting of the {}.',
+        'the plastic {}.',
+        'a sculpture of the {}.',
+        'a sketch of the {}.',
+        'a tattoo of the {}.',
+        'the toy {}.',
+        'a rendition of the {}.',
+        'the embroidered {}.',
+        'the cartoon {}.',
+        'the {} in a video game.',
+        'the plushie {}.',
+        'the origami {}.',
+        'art of the {}.',
+        'graffiti of the {}.',
+        'a drawing of the {}.',
+        'a doodle of the {}.',
+    ]
+}
+
+Food101_CLASSES_TEMPLATES = {
+    'classes' : [
+        'apple pie',
+        'baby back ribs',
+        'baklava',
+        'beef carpaccio',
+        'beef tartare',
+        'beet salad',
+        'beignets',
+        'bibimbap',
+        'bread pudding',
+        'breakfast burrito',
+        'bruschetta',
+        'caesar salad',
+        'cannoli',
+        'caprese salad',
+        'carrot cake',
+        'ceviche',
+        'cheese plate',
+        'cheesecake',
+        'chicken curry',
+        'chicken quesadilla',
+        'chicken wings',
+        'chocolate cake',
+        'chocolate mousse',
+        'churros',
+        'clam chowder',
+        'club sandwich',
+        'crab cakes',
+        'creme brulee',
+        'croque madame',
+        'cup cakes',
+        'deviled eggs',
+        'donuts',
+        'dumplings',
+        'edamame',
+        'eggs benedict',
+        'escargots',
+        'falafel',
+        'filet mignon',
+        'fish and chips',
+        'foie gras',
+        'french fries',
+        'french onion soup',
+        'french toast',
+        'fried calamari',
+        'fried rice',
+        'frozen yogurt',
+        'garlic bread',
+        'gnocchi',
+        'greek salad',
+        'grilled cheese sandwich',
+        'grilled salmon',
+        'guacamole',
+        'gyoza',
+        'hamburger',
+        'hot and sour soup',
+        'hot dog',
+        'huevos rancheros',
+        'hummus',
+        'ice cream',
+        'lasagna',
+        'lobster bisque',
+        'lobster roll sandwich',
+        'macaroni and cheese',
+        'macarons',
+        'miso soup',
+        'mussels',
+        'nachos',
+        'omelette',
+        'onion rings',
+        'oysters',
+        'pad thai',
+        'paella',
+        'pancakes',
+        'panna cotta',
+        'peking duck',
+        'pho',
+        'pizza',
+        'pork chop',
+        'poutine',
+        'prime rib',
+        'pulled pork sandwich',
+        'ramen',
+        'ravioli',
+        'red velvet cake',
+        'risotto',
+        'samosa',
+        'sashimi',
+        'scallops',
+        'seaweed salad',
+        'shrimp and grits',
+        'spaghetti bolognese',
+        'spaghetti carbonara',
+        'spring rolls',
+        'steak',
+        'strawberry shortcake',
+        'sushi',
+        'tacos',
+        'takoyaki',
+        'tiramisu',
+        'tuna tartare',
+        'waffles',
+    ],
+
+    'templates' : [
+        'a photo of {}, a type of food.',
+    ]
+}
+
+data_seeds = [107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121]

vlm_eval/ms_coco_gen.py

@@ -0,0 +1,76 @@
+import os
+import json
+import torchvision.datasets as dset
+import torchvision.transforms as transforms
+from coco_cf import COCO_CF_dataset
+from torch.utils.data import DataLoader
+
+def custom_collate_fn(batch):
+    collated_batch = {}
+    for key in batch[0].keys():
+        collated_batch[key] = [item[key] for item in batch]
+    return collated_batch
+
+coco_2017 = dset.CocoCaptions(root='./open_flamingo_datasets/COCO_2017/val2017/',
+                              annFile='./open_flamingo_datasets/COCO_2017/captions_val2017.json',
+                              transform=transforms.ToTensor())
+
+coco_cf = COCO_CF_dataset(base_dir='./open_flamingo_datasets/COCO_CF/')
+dl_coco_cf = DataLoader(coco_cf, batch_size=100,collate_fn=custom_collate_fn)
+
+
+# Collect both captions from each batch in one step
+coco_cf_captions = []
+
+for batch in dl_coco_cf:
+    # Extend the list with both captions at once without list comprehension
+    coco_cf_captions.extend([caption.replace('.','').replace(",","").replace("-"," ").replace("'s","").lower().strip() for caption in batch['caption_0']])
+
+ms_coco_gen_indices = []
+coco_cf_captions_set = set(coco_cf_captions)
+
+for index in range(len(coco_2017)):
+    image_id = coco_2017.ids[index]
+    _,captions = coco_2017[index]
+
+
+    matches = [s for s in captions if s.replace(".","").replace(",","").replace("'s","").replace("-"," ").lower().strip() in coco_cf_captions_set]
+
+
+    for match in matches:
+        ms_coco_gen_indices.append((image_id,match))
+ms_coco_gen_indices = ms_coco_gen_indices[:17410]
+print(ms_coco_gen_indices)
+ms_coco = [{'image_id': image_index,'caption': caption} for (image_index, caption) in ms_coco_gen_indices]
+
+file_path = 'ms_coco_captions.json'
+
+# Save the dictionary to a JSON file
+
+import os
+
+# Base path where the images are located
+base_image_path = '/home/kc/Downloads/val2017/'
+
+# Assuming ms_coco_gen_indices is a list of (image_index, caption) tuples
+ms_coco_gen_indices = [(image_index, caption) for (image_index, caption) in ms_coco_gen_indices]
+
+# List to store the updated entries with pathtoimage included
+updated_ms_coco_gen_indices = []
+
+# Process each (image_index, caption) in ms_coco_gen_indices
+for image_index, caption in ms_coco_gen_indices:
+    # Construct the full path to the image file based on the image_index
+    pathtoimage = f"{image_index:012d}.jpg"  # Ensure image_index is 12 digits with padding
+
+    # Append the new entry as (image_index, pathtoimage, caption)
+    updated_ms_coco_gen_indices.append((image_index, pathtoimage, caption))
+
+# Now ms_coco_gen_indices includes (image_index, pathtoimage, caption)
+ms_coco_gen_indices = updated_ms_coco_gen_indices
+ms_coco = [{'image_id': image_index,'image_name': image_name,'caption': caption} for (image_index,image_name ,caption) in ms_coco_gen_indices]
+
+with open(file_path, 'w') as json_file:
+    for row in ms_coco:
+        json.dump(row, json_file)
+        json_file.write('\n')