from PIL import Image import numpy as np import torch import torch.nn as nn import torch.optim as optim from torchvision import transforms, models IMG_MEAN = [0.485, 0.456, 0.406] IMG_STD = [0.229, 0.224, 0.225] CLIP_MEAN = [0.48145466, 0.4578275, 0.40821073] CLIP_STD = [0.26862954, 0.26130258, 0.27577711] DEVICE = "cuda" if torch.cuda.is_available() else "cpu" LR = 5e-4 CROP_SIZE = 128 RESIZE = 224 NUM_EPOCHS = 200 NUM_CROPS = 64 PATCH_THRESHOLD = 0.7 L_TV = 2e-3 L_PATCH = 9000 L_DIR = 500 L_CONTENT = 150 IMG_SIZE = 512 # copied from openai clip IMAGENET_TEMPLATES = [ "a bad photo of a {}.", "a photo of many {}.", "a sculpture of a {}.", "a photo of the hard to see {}.", "a low resolution photo of the {}.", "a rendering of a {}.", "graffiti of a {}.", "a bad photo of the {}.", "a cropped photo of the {}.", "a tattoo of a {}.", "the embroidered {}.", "a photo of a hard to see {}.", "a bright photo of a {}.", "a photo of a clean {}.", "a photo of a dirty {}.", "a dark photo of the {}.", "a drawing of a {}.", "a photo of my {}.", "the plastic {}.", "a photo of the cool {}.", "a close-up photo of a {}.", "a black and white photo of the {}.", "a painting of the {}.", "a painting of a {}.", "a pixelated photo of the {}.", "a sculpture of the {}.", "a bright photo of the {}.", "a cropped photo of a {}.", "a plastic {}.", "a photo of the dirty {}.", "a jpeg corrupted photo of a {}.", "a blurry photo of the {}.", "a photo of the {}.", "a good photo of the {}.", "a rendering of the {}.", "a {} in a video game.", "a photo of one {}.", "a doodle of a {}.", "a close-up photo of the {}.", "a photo of a {}.", "the origami {}.", "the {} in a video game.", "a sketch of a {}.", "a doodle of the {}.", "a origami {}.", "a low resolution photo of a {}.", "the toy {}.", "a rendition of the {}.", "a photo of the clean {}.", "a photo of a large {}.", "a rendition of a {}.", "a photo of a nice {}.", "a photo of a weird {}.", "a blurry photo of a {}.", "a cartoon {}.", "art of a {}.", "a sketch of the {}.", "a embroidered {}.", "a pixelated photo of a {}.", "itap of the {}.", "a jpeg corrupted photo of the {}.", "a good photo of a {}.", "a plushie {}.", "a photo of the nice {}.", "a photo of the small {}.", "a photo of the weird {}.", "the cartoon {}.", "art of the {}.", "a drawing of the {}.", "a photo of the large {}.", "a black and white photo of a {}.", "the plushie {}.", "a dark photo of a {}.", "itap of a {}.", "graffiti of the {}.", "a toy {}.", "itap of my {}.", "a photo of a cool {}.", "a photo of a small {}.", "a tattoo of the {}.", ] def get_mean(mean_dist): mean = torch.tensor(mean_dist).to(DEVICE) return mean.view(1, -1, 1, 1) def get_std(std_dist): std = torch.tensor(std_dist).to(DEVICE) return std.view(1, -1, 1, 1) def normalize(data): mean = get_mean(IMG_MEAN) std = get_std(IMG_STD) norm_data = (data - mean) / std return norm_data def clip_normalize(data): resized = nn.functional.interpolate(data, size=RESIZE, mode='bicubic') mean = get_mean(CLIP_MEAN) std = get_std(CLIP_STD) norm_data = (resized - mean) / std return norm_data def load_image(image): image = image.resize((IMG_SIZE, IMG_SIZE)) transform = transforms.Compose([transforms.ToTensor()]) return transform(image)[:3, :, :].unsqueeze(0) def get_features(image, vgg19): # uses vgg19 model to extract content features layers = {'0': 'conv1_1', '5': 'conv2_1', '10': 'conv3_1', '19': 'conv4_1', '21': 'conv4_2', '28': 'conv5_1', '31': 'conv5_2' } features = {} x = image for name, layer in vgg19._modules.items(): x = layer(x) if name in layers: features[layers[name]] = x return features def prompt_ensemble(prompt): return [template.format(prompt) for template in IMAGENET_TEMPLATES] def get_image_prior_losses(target): diff1 = target[:, :, :, :-1] - target[:, :, :, 1:] diff2 = target[:, :, :-1, :] - target[:, :, 1:, :] diff3 = target[:, :, 1:, :-1] - target[:, :, :-1, 1:] diff4 = target[:, :, :-1, :-1] - target[:, :, 1:, 1:] loss_var_l2 = torch.norm(diff1) + torch.norm(diff2) + torch.norm(diff3) + torch.norm(diff4) return loss_var_l2