Spaces:
Running on Zero
Running on Zero
File size: 5,291 Bytes
151f388 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 | import time
from torch.utils.data import Dataset, DataLoader
import numpy as np
import torch
from data.dataset import AIGCDetectionDataset
from data.transform import create_train_transforms, create_val_transforms
def merge_tensor(img, label, is_train=True):
def shuffle_tensor(img, label):
indices = torch.randperm(img.size(0))
return img[indices], label[indices]
if isinstance(img, list) and isinstance(label, list):
img, label = torch.cat(img, dim=0), torch.cat(label, dim=0)
if is_train:
img, label = shuffle_tensor(img, label)
return img, label
def debug_data_loader():
input_size = 224
use_label = True
is_crop = True
is_dire = False
phase = 'train'
fake_indexes = '2'
inpainting_dir = 'full_inpainting'
# DRCT-2M
# root_path = '/disk1/chenby/dataset/AIGC_data/DRCT_data/MSCOCO'
# fake_root_path = '/disk1/chenby/dataset/AIGC_data/DRCT_data/DRCT-2M'
# GenImage
# root_path = '/disk1/chenby/dataset/AIGC_data/DRCT_data/GenImage'
# fake_root_path = ''
# DIRE of DRCT-2M
# inpainting_dir = 'full_inpainting'
# is_dire = True
# root_dir = '/disk1/chenby/dataset/AIGC_data/DRCT_data'
# root_path = f'{root_dir}/MSCOCO,{root_dir}/DR/MSCOCO'
# fake_root_path = f'{root_dir}/DRCT-2M,{root_dir}/DR/DRCT-2M'
# DIRE of GenImage
# fake_indexes = '1'
# is_dire = True
# inpainting_dir = 'inpainting'
# root_dir = '/disk1/chenby/dataset/AIGC_data/DRCT_data'
# root_path = f'{root_dir}/DR/GenImage'
# fake_root_path = f'{root_dir}/DR/GenImage'
# DRCT of DRCT-2M
# root_dir = '/disk1/chenby/dataset/AIGC_data/DRCT_data'
# root_path = f'{root_dir}/MSCOCO/train2017'
# fake_root_path = f'{root_dir}/DR/MSCOCO/full_inpainting/train2017,' \
# f'{root_dir}/DRCT-2M/stable-diffusion-v1-4/train2017,' \
# f'{root_dir}/DR/DRCT-2M/stable-diffusion-v1-4/full_inpainting/train2017'
# DRCT of GenImage
fake_indexes = '1'
is_dire = False
root_dir = '/disk1/chenby/dataset/AIGC_data/DRCT_data'
root_path = f'{root_dir}/GenImage/stable_diffusion_v_1_5/imagenet_ai_0424_sdv5/train/nature'
fake_root_path = f'{root_dir}/DR/GenImage/stable_diffusion_v_1_4/imagenet_ai_0419_sdv4/train/nature/inpainting,' \
f'{root_dir}/GenImage/stable_diffusion_v_1_5/imagenet_ai_0424_sdv5/train/ai,' \
f'{root_dir}/DR/GenImage/stable_diffusion_v_1_4/imagenet_ai_0419_sdv4/train/ai/inpainting'
transform = create_train_transforms(size=input_size, is_crop=is_crop)
xdl = AIGCDetectionDataset(root_path=root_path, fake_root_path=fake_root_path, phase=phase, use_label=use_label,
transform=transform, fake_indexes=fake_indexes, is_dire=is_dire,
inpainting_dir=inpainting_dir,
)
train_loader = DataLoader(xdl, batch_size=32, shuffle=True, num_workers=8)
print('length:', len(xdl), f'iter:{len(train_loader)}')
start = time.time()
times = 10
for i, data in enumerate(train_loader):
if use_label:
(img, label) = data
if isinstance(img, list) and not isinstance(label, list):
print(i, img[0].size(), img[1].size(), label.size(), label)
else:
img, label = merge_tensor(img, label, is_train=True)
print(i, img.size(), label.size(), label)
else:
img, image_name = data
if isinstance(img, list):
print(f'{i + 1}/{len(train_loader)}', img[0].size(), img[1].size(), len(image_name))
else:
print(f'{i + 1}/{len(train_loader)}', img.size(), len(image_name))
if i == times - 1:
break
end = time.time()
QPS = times / (end - start)
print(f"run_time:{end - start}s, QPS:{QPS}")
def show_image():
import cv2
from data.dataset import read_image
image_path = 'data/samples/01.png'
img, _ = read_image(image_path)
is_success, img_bytes = cv2.imencode(".png", img) # .jpg有损,.png无损
img_bytes = img_bytes.tobytes()
# 将 bytes 类型的图片数据恢复为 OpenCV 图像
img_array = np.frombuffer(img_bytes, dtype=np.uint8)
img_decoded = cv2.imdecode(img_array, cv2.IMREAD_COLOR)
print(img.shape)
diff = np.abs(img - img_decoded)
print(np.mean(diff), np.max(diff))
def test_post_aug():
from data.dataset import read_image, cv2_jpg
import matplotlib.pyplot as plt
image_path = 'data/samples/01.png'
img, _ = read_image(image_path)
img_aug = cv2_jpg(img, compress_val=100)
diff = np.abs(img - img_aug).astype(np.uint8)
# print(np.mean(diff), np.max(diff))
# 创建一个新的figure
plt.figure(figsize=(15, 5))
# 画出原始图像
plt.subplot(1, 3, 1)
plt.imshow(img)
plt.title('Original Image')
# 画出后处理的图像
plt.subplot(1, 3, 2)
plt.imshow(img_aug)
plt.title('Image after Post Aug.')
# 画出后处理前后的图像差异
plt.subplot(1, 3, 3)
plt.imshow(diff)
plt.title('Diff after Post Aug.')
# 显示图像
plt.show()
if __name__ == '__main__':
debug_data_loader()
|