AnimeRun/segmat/datasets/utils/augmentor.py

import numpy as np
import random
import math
from PIL import Image

import cv2
cv2.setNumThreads(0)
cv2.ocl.setUseOpenCL(False)

import torch
from torchvision.transforms import ColorJitter
import torch.nn.functional as F
from collections import Counter

class SegMatAugmentor:
    def __init__(self, crop_size, min_scale=-0.2, max_scale=0.5, do_flip=True):
        
        # spatial augmentation params
        self.crop_size = crop_size
        self.min_scale = min_scale
        self.max_scale = max_scale
        self.spatial_aug_prob = 0.8
        self.stretch_prob = 0.8
        self.max_stretch = 0.2

        # flip augmentation params
        self.do_flip = do_flip
        self.h_flip_prob = 0.5
        self.v_flip_prob = 0.1

        # photometric augmentation params
        self.photo_aug = ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4, hue=0.5/3.14)
        self.asymmetric_color_aug_prob = 0.2
        self.eraser_aug_prob = 0.5

    def color_transform(self, img1, img2):
        """ Photometric augmentation """

        # asymmetric
        if np.random.rand() < self.asymmetric_color_aug_prob:
            img1 = np.array(self.photo_aug(Image.fromarray(img1)), dtype=np.uint8)
            img2 = np.array(self.photo_aug(Image.fromarray(img2)), dtype=np.uint8)

        # symmetric
        else:
            image_stack = np.concatenate([img1, img2], axis=0)
            image_stack = np.array(self.photo_aug(Image.fromarray(image_stack)), dtype=np.uint8)
            img1, img2 = np.split(image_stack, 2, axis=0)

        return img1, img2

    def eraser_transform(self, img1, img2, bounds=[50, 100]):
        """ Occlusion augmentation """

        ht, wd = img1.shape[:2]
        if np.random.rand() < self.eraser_aug_prob:
            mean_color = np.mean(img2.reshape(-1, 3), axis=0)
            for _ in range(np.random.randint(1, 3)):
                x0 = np.random.randint(0, wd)
                y0 = np.random.randint(0, ht)
                dx = np.random.randint(bounds[0], bounds[1])
                dy = np.random.randint(bounds[0], bounds[1])
                img2[y0:y0+dy, x0:x0+dx, :] = mean_color

        return img1, img2

    def spatial_transform(self, img1, img2, seg1, seg2, matching):
        # augmentation should contain crop, shift, scaling
        # when croped: seg index need to be changed to [0..Nc], lost item mismatching
        # when scaled: seg index need to be rescaled. if larger, doesn't matter, if smaller, index may vanish. delete it and reindex as above
        # when shift: doesn't matter
        # randomly sample scale
        ht, wd = img1.shape[:2]
        min_scale = np.maximum(
            (self.crop_size[0] + 8) / float(ht), 
            (self.crop_size[1] + 8) / float(wd))

        scale = 2 ** np.random.uniform(self.min_scale, self.max_scale)
        scale_x = scale
        scale_y = scale
        
        x = np.random.rand()
        # print(x, x, x, x, x)
        if x < self.stretch_prob:
            scale_x *= 2 ** np.random.uniform(-self.max_stretch, self.max_stretch)
            scale_y *= 2 ** np.random.uniform(-self.max_stretch, self.max_stretch)
        
        scale_x = np.clip(scale_x, min_scale, None)
        scale_y = np.clip(scale_y, min_scale, None)

        if np.random.rand() < self.spatial_aug_prob:
            # rescale the images
            img1 = cv2.resize(img1, None, fx=scale_x, fy=scale_y, interpolation=cv2.INTER_LINEAR)
            img2 = cv2.resize(img2, None, fx=scale_x, fy=scale_y, interpolation=cv2.INTER_LINEAR)
            
            seg1 = cv2.resize(seg1, None, fx=scale_x, fy=scale_y, interpolation=cv2.INTER_NEAREST)
            seg2 = cv2.resize(seg2, None, fx=scale_x, fy=scale_y, interpolation=cv2.INTER_NEAREST)

        if self.do_flip:
            if np.random.rand() < self.h_flip_prob: # h-flip
                img1 = img1[:, ::-1]
                img2 = img2[:, ::-1]
                seg1 = seg1[:, ::-1]
                seg2 = seg2[:, ::-1]

            if np.random.rand() < self.v_flip_prob: # v-flip
                img1 = img1[::-1, :]
                img2 = img2[::-1, :]
                seg1 = seg1[::-1, :]
                seg2 = seg2[::-1, :]

        y0 = np.random.randint(0, img1.shape[0] - self.crop_size[0])
        x0 = np.random.randint(0, img1.shape[1] - self.crop_size[1])
        
        img1 = img1[y0:y0+self.crop_size[0], x0:x0+self.crop_size[1]]
        img2 = img2[y0:y0+self.crop_size[0], x0:x0+self.crop_size[1]]
        seg1 = seg1[y0:y0+self.crop_size[0], x0:x0+self.crop_size[1]]
        seg2 = seg2[y0:y0+self.crop_size[0], x0:x0+self.crop_size[1]]

        seg1_id_hist = Counter(seg1.reshape(-1)).most_common()
        seg2_id_hist = Counter(seg2.reshape(-1)).most_common()

        # cc_5 = 0
        # for ii, jj in seg1_id_hist:
        #     if jj < 10:
        #         cc_5 += 1
        # print('# < 5 is ', cc_5)

        seg1_id = np.array([ii for (ii, _) in seg1_id_hist])
        seg2_id = np.array([ii for (ii, _) in seg2_id_hist])

        seg1_new = seg1.copy()
        seg2_new = seg2.copy()
        for ii in range(len(seg1_id)):
            seg1_new[seg1 == seg1_id[ii]] = ii 
        for ii in range(len(seg2_id)):
            seg2_new[seg2 == seg2_id[ii]] = ii

        matching_new = np.ones([len(seg1_id), ]) * -1
        for ii in range(len(seg1_id)):
            if matching[seg1_id[ii]] in seg2_id:
                matching_new[ii] = np.where(seg2_id == matching[seg1_id[ii]])[0][0] 

        return img1, img2, seg1_new, seg2_new, matching_new

    def __call__(self, img1, img2, seg1, seg2, matching):
        img1, img2 = self.color_transform(img1, img2)
        img1, img2 = self.eraser_transform(img1, img2)
        img1, img2, seg1, seg2, matching = self.spatial_transform(img1, img2, seg1, seg2, matching)

        img1 = np.ascontiguousarray(img1)
        img2 = np.ascontiguousarray(img2)
        seg1 = np.ascontiguousarray(seg1)
        seg2 = np.ascontiguousarray(seg2)
        matching = np.ascontiguousarray(matching)

        return img1, img2, seg1, seg2, matching