import torch.nn as nn from taming.models.vqgan import VQModel, VQModel_w_Prompt import torch.nn.functional as F def pad_to_multiple_of_16(latent, pad_value, patch_size=16): h, w = latent.size(2), latent.size(3) target_h = ((h - 1) // patch_size + 1) * patch_size target_w = ((w - 1) // patch_size + 1) * patch_size pad_h = (target_h - h) // 2 pad_w = (target_w - w) // 2 # 额外处理奇数padding的情况 pad_h_extra = (target_h - h) % 2 pad_w_extra = (target_w - w) % 2 padded_latent = F.pad(latent, (pad_w, pad_w + pad_w_extra, pad_h, pad_h + pad_h_extra), mode='constant', value=pad_value) # 指定左、右、上、下的填充宽度 print("After padding: ", padded_latent.shape) return padded_latent def split_into_blocks(latent, patch_size=16): b, c, h, w = latent.size() blocks = latent.view(b, c, h // patch_size, patch_size, w // patch_size, patch_size) blocks = blocks.permute(0, 2, 4, 1, 3, 5).contiguous().view(-1, c, patch_size, patch_size) print("After splitting into blocks: ", blocks.shape) return blocks # def merge_blocks(blocks, original_shape): # b, c, h, w = original_shape # num_blocks_per_row = w // 16 # num_blocks_per_col = h // 16 # # 恢复到原始形状的顺序 # blocks = blocks.view(b, num_blocks_per_col, num_blocks_per_row, c, 16, 16) # blocks = blocks.permute(0, 3, 1, 4, 2, 5).contiguous() # blocks = blocks.view(b, c, h, w) # print("After merging blocks: ", blocks.shape) # return blocks def merge_blocks(blocks, original_shape, patch_size=16): b, c, h, w = original_shape num_blocks_per_row = w // patch_size num_blocks_per_col = h // patch_size # 恢复到原始形状的顺序 blocks = blocks.view(b, num_blocks_per_col, num_blocks_per_row, c, patch_size, patch_size) blocks = blocks.permute(0, 3, 1, 4, 2, 5).contiguous() blocks = blocks.view(b, c, h, w) print("After merging blocks: ", blocks.shape) return blocks def crop_to_original_shape(blocks, original_shape): _, _, padded_height, padded_width = blocks.shape original_height, original_width = original_shape[2], original_shape[3] start_h = (padded_height - original_height) // 2 end_h = start_h + original_height start_w = (padded_width - original_width) // 2 end_w = start_w + original_width cropped_blocks = blocks[:, :, start_h:end_h, start_w:end_w] print("After cropping to original shape: ", cropped_blocks.shape) return cropped_blocks class Model_VQ(nn.Module): def __init__(self, ddconfig, n_embed, embed_dim, ckpt_path): super(Model_VQ, self).__init__() self.vqgan = VQModel_w_Prompt(ddconfig=ddconfig, n_embed=n_embed, embed_dim=embed_dim, ckpt_path=ckpt_path) # self.mask_token_label = 2024 # for param in self.vqgan.parameters(): # param.requires_grad = False for name, param in self.vqgan.named_parameters(): if 'prompt' not in name: param.requires_grad = False def forward(self, input): # codebook_emb_dim = 256 z_q, _, token_tuple = self.vqgan.encode(input) # z_q: (b0, 256, h0, w0), token_tuple: (B, 256, h0, w0) gen_images =self.vqgan.decode(z_q) return gen_images class Model_VQ_former(nn.Module): def __init__(self, ddconfig, n_embed, embed_dim, ckpt_path): super(Model_VQ_former, self).__init__() # self.vqgan = VQModel_w_Prompt(ddconfig=ddconfig, n_embed=n_embed, embed_dim=embed_dim, ckpt_path=ckpt_path) self.vqgan = VQModel(ddconfig=ddconfig, n_embed=n_embed, embed_dim=embed_dim, ckpt_path=ckpt_path) # self.mask_token_label = 2024 # for param in self.vqgan.parameters(): # param.requires_grad = False for name, param in self.vqgan.named_parameters(): if 'prompt' not in name: param.requires_grad = False def forward(self, input): # codebook_emb_dim = 256 z_q, _, token_tuple = self.vqgan.encode(input) # z_q: (b0, 256, h0, w0), token_tuple: (B, 256, h0, w0) gen_images =self.vqgan.decode(z_q) return gen_images if __name__ == "__main__": import torchvision.transforms as transforms from PIL import Image import matplotlib.pyplot as plt # 加载和处理图像 img = Image.open('/home/t2vg-a100-G4-10/project/qyp/mimc_rope/shark/val/rec/000000001000.jpg').convert('RGB') # 修改为你的图像路径 transform = transforms.Compose([ transforms.ToTensor(), ]) img_tensor = transform(img).unsqueeze(0) # 添加批次维度 # 应用函数 padded_img = pad_to_multiple_of_16(img_tensor, pad_value=0, patch_size=256) blocks = split_into_blocks(padded_img, patch_size=256) # 可视化和保存块 for i, block in enumerate(blocks): plt.imshow(block.permute(1, 2, 0).numpy()) plt.title(f'Block {i}') plt.savefig(f'block_{i}.png') # 保存每个块的图片