maskrcnn_benchmark/data/transforms/transforms.py

# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
import random

import torch
import torchvision
from torchvision.transforms import functional as F

from maskrcnn_benchmark.structures.bounding_box import BoxList
from maskrcnn_benchmark.structures.segmentation_mask import SegmentationMask

from maskrcnn_benchmark.structures.ke import textKES
from maskrcnn_benchmark.structures.mty import MTY
import numpy as np
from PIL import Image
from shapely.geometry import *
import cv2
from maskrcnn_benchmark.config import cfg


class Compose(object):
    def __init__(self, transforms):
        self.transforms = transforms

    def __call__(self, image, target):
        for t in self.transforms:
            image, target = t(image, target)
        return image, target

    def __repr__(self):
        format_string = self.__class__.__name__ + "("
        for t in self.transforms:
            format_string += "\n"
            format_string += "    {0}".format(t)
        format_string += "\n)"
        return format_string


class Resize(object):
    def __init__(self, min_size, max_size):
        if not isinstance(min_size, (list, tuple)):
            min_size = (min_size,)
        self.min_size = min_size
        self.max_size = max_size

    # modified from torchvision to add support for max size
    def get_size(self, image_size):

       # if test ic15

       #oh = 1200
       #ow = 2000
       #return (oh, ow)

       w, h = image_size
       size = random.choice(self.min_size)
       max_size = self.max_size
       if max_size is not None:
           min_original_size = float(min((w, h)))
           max_original_size = float(max((w, h)))
           if max_original_size / min_original_size * size > max_size:
               size = int(round(max_size * min_original_size / max_original_size))

       if (w <= h and w == size) or (h <= w and h == size):
           return (h, w)

       if w < h:
           ow = size
           oh = int(size * h / w)
       else:
           oh = size
           ow = int(size * w / h)

       return (oh, ow)

    def __call__(self, image, target):
        size = self.get_size(image.size)
        image = F.resize(image, size)
        if isinstance(target, list):
            target = [t.resize(image.size) for t in target]
        else:
            target = target.resize(image.size)
        return image, target


class RandomHorizontalFlip(object):
    def __init__(self, prob=0.5):
        self.prob = prob

    def __call__(self, image, target):
        if random.random() < self.prob:
            image = F.hflip(image)
            if isinstance(target, list):
                target = [t.transpose(0) for t in target]
            else:
                target = target.transpose(0)
        return image, target


class ToTensor(object):
    def __call__(self, image, target):
        return F.to_tensor(image), target


class Normalize(object):
    def __init__(self, mean, std, to_bgr255=True):
        self.mean = mean
        self.std = std
        self.to_bgr255 = to_bgr255

    def __call__(self, image, target):
        if self.to_bgr255:
            image = image[[2, 1, 0]] * 255
        image = F.normalize(image, mean=self.mean, std=self.std)
        return image, target


class RandomCrop(object):
    """Random crop with repeatedly expanding the range to included box borders."""
    def __init__(self, prob, init_crop_size=(0.5, 1.0)):

        if (not isinstance(init_crop_size, list)) and (not isinstance(init_crop_size, tuple)):
            raise ValueError('Paremeter init_crop_size should be a list or tuple!')
        elif len(init_crop_size) != 2:
            raise ValueError('Length of init_crop_size should be 2!')
        elif not (init_crop_size[0] <= 1 and init_crop_size[0] >= 0 and init_crop_size[1] <= 1 and init_crop_size[1] >= 0):
            raise ValueError('Elements of init_crop_size should be within [0, 1]!')
        self.prob = prob
        self.init_crop_size = init_crop_size

    def __call__(self, image, target):
        if random.random() >= self.prob:
            return image, target

        if isinstance(target, list):
            target0 = target[0]
        else:
            target0 = target
        while True:
            # Initial Crop Region
            crop_region = self.initial_crop_region(image)

            # Adjust Crop Region
            crop_region, keep_target = self.adjust_crop_region(crop_region, target0)
            if crop_region is None and keep_target is None:
                continue

            if isinstance(target, list):
                # check empty char
                new_t1 = target[1].crop(crop_region)
                if len(new_t1) < 1: return image, target

            image = image.crop(crop_region.numpy())
            if isinstance(target, list):
                target0 = keep_target.crop(crop_region)
                others = [t.crop(crop_region, remove_empty=True) for t in target[1:]]
                target = [target0] + others
            else:
                target = keep_target.crop(crop_region)

            return image, target

    def initial_crop_region(self, image):
        width, height = image.size
        ratio_w, ratio_h = torch.empty(2).uniform_(self.init_crop_size[0], self.init_crop_size[1])
        crop_width, crop_height = int(width*ratio_w), int(height*ratio_h)
        crop_xmin = torch.randint(width-crop_width, (1,))
        crop_ymin = torch.randint(height-crop_height, (1,))
        crop_xmax = crop_xmin + crop_width
        crop_ymax = crop_ymin + crop_height
        crop_region = torch.Tensor([crop_xmin, crop_ymin, crop_xmax, crop_ymax])
        return crop_region

    def intersect_area(self, bbox, bboxes):
        inter_xmin = torch.max(bbox[0], bboxes[:, 0])
        inter_ymin = torch.max(bbox[1], bboxes[:, 1])
        inter_xmax = torch.min(bbox[2], bboxes[:, 2])
        inter_ymax = torch.min(bbox[3], bboxes[:, 3])
        inter_width = torch.max(torch.Tensor([0]), inter_xmax-inter_xmin)
        inter_height = torch.max(torch.Tensor([0]), inter_ymax-inter_ymin)
        return inter_width*inter_height

    def adjust_crop_region(self, crop_region, target):
        keep_indies_ = torch.zeros((len(target)), dtype=torch.uint8)
        while True:
            inter_area = self.intersect_area(crop_region, target.bbox)
            keep_indies = (inter_area > 0)
            if torch.sum(keep_indies) == 0:
                return None, None
            keep_target = target[keep_indies]
            if keep_indies.equal(keep_indies_):
                return crop_region, keep_target
            keep_bbox = keep_target.bbox
            crop_xmin = torch.min(crop_region[0], torch.min(keep_bbox[:, 0]))
            crop_ymin = torch.min(crop_region[1], torch.min(keep_bbox[:, 1]))
            crop_xmax = torch.max(crop_region[2], torch.max(keep_bbox[:, 2]))
            crop_ymax = torch.max(crop_region[3], torch.max(keep_bbox[:, 3]))
            crop_region = torch.Tensor([crop_xmin, crop_ymin, crop_xmax, crop_ymax])
            keep_indies_ = keep_indies

class RandomBrightness(object):
    def __init__(self, prob=0.5):
        self.prob = prob

    def __call__(self, image, target):
        if random.random() < self.prob:
            brightness_factor = random.uniform(0.5, 2)
            image = F.adjust_brightness(image, brightness_factor)
        return image, target

class RandomContrast(object):
    def __init__(self, prob=0.5):
        self.prob = prob

    def __call__(self, image, target):
        if random.random() < self.prob:
            contrast_factor = random.uniform(0.5, 2)
            image = F.adjust_contrast(image, contrast_factor)
        return image, target

class RandomHue(object):
    def __init__(self, prob=0.5):
        self.prob = prob

    def __call__(self, image, target):
        if random.random() < self.prob:
            hue_factor = random.uniform(-0.25, 0.25)
            image = F.adjust_hue(image, hue_factor)
        return image, target

class RandomSaturation(object):
    def __init__(self, prob=0.5):
        self.prob = prob

    def __call__(self, image, target):
        if random.random() < self.prob:
            saturation_factor = random.uniform(0.5, 2)
            image = F.adjust_saturation(image, saturation_factor)
        return image, target

class RandomGamma(object):
    def __init__(self, prob=0.5):
        self.prob = prob

    def __call__(self, image, target):
        if random.random() < self.prob:
            gamma_factor = random.uniform(0.5, 2)
            image = F.adjust_gamma(image, gamma_factor)
        return image, target


class RandomRotation(object):
    def __init__(self, prob = 0.3, degree = 5):
        self.prob = prob
        self.degree = degree

    def kes_encode(self, kes):
        kes_encode = []
        for i in kes:
            mnx = i[0]
            mny = i[1]
            assert(len(i)%3 == 0)
            npts = int(len(i)/3-2)
            for index in range(npts):
                i[3+index*3]  = (i[3+index*3]+mnx)/2
                i[4+index*3]  = (i[4+index*3]+mny)/2
            kes_encode.append(i)
        return kes_encode
    
    def kes_gen(self, kes):
        kes_gen_out = []
        for i in kes:
            mnx = i[0]
            mny = i[1]
            cx= i[27]
            cy= i[28]
            assert(len(i)%3 == 0)
            ot = [mnx, i[3],i[6],i[9],i[12], cx,\
                  mny, i[16],i[19],i[22],i[25], cy]
            kes_gen_out.append(ot)
        return kes_gen_out

    def __call__(self, image, target):
        if random.random() < self.prob:
            image1 = image
            target1 = target
            img = np.array(image)
            w = image.size[0]
            h = image.size[1]
            pri_points = []
            for i in range(len(target.extra_fields['masks'].instances)):
                assert(len(target.extra_fields['masks'].instances[i].polygons)==1), 'one text instance should have only one polygon.'
                tensor_box = target.extra_fields['masks'].instances[i].polygons[0].polygons

                points_x = np.array([tensor_box[0][0],tensor_box[0][2],tensor_box[0][4],tensor_box[0][6]])
                points_y = np.array([tensor_box[0][1],tensor_box[0][3],tensor_box[0][5],tensor_box[0][7]])
                smaller_x = np.where(points_x <= 0)
                larger_x = np.where(points_x >= w)
                smaller_y = np.where(points_y <= 0)
                larger_y = np.where(points_y >= h)
                points_x[smaller_x] = 1
                points_x[larger_x] = w - 1
                points_y[smaller_y] = 1
                points_y[larger_y] = h -1
                pri_points.append((int(points_x[0]),int(points_y[0])))
                pri_points.append((int(points_x[1]),int(points_y[1])))
                pri_points.append((int(points_x[2]),int(points_y[2])))
                pri_points.append((int(points_x[3]),int(points_y[3])))

            #get the transform image and points  
            height, width = img.shape[:2]

            # if ROTATE_DEGREE = (0,30,60,90,210,150,180,210,240,270,300,330,360)
            #de_ro = random.choice(self.degree)
            #matrix = cv2.getRotationMatrix2D((width / 2, height / 2) ,de_ro, 1.0)

            # if ROTATE_DEGREE = 10
            matrix = cv2.getRotationMatrix2D((width / 2, height / 2), random.uniform(-self.degree[0],self.degree[0]), 1.0)

            cos = np.abs(matrix[0,0])
            sin = np.abs(matrix[0,1])
            new_W = int((height * sin) + (width * cos))
            new_H = int((height * cos) + (width * sin))
            matrix[0,2] += (new_W/2) - width/2
            matrix[1,2] += ((new_H/2)) - height/2
            img = cv2.warpAffine(img, matrix, (new_W,new_H))

            change_points = []
            for i in range(int(len(pri_points))):
                x_r,y_r = cv2.transform(np.array([[pri_points[i]]]),matrix).squeeze()
                change_points.append([x_r,y_r])
        
            image = Image.fromarray(img)

            keypoints_len = len(change_points)
            tran_boxes = []
            n = keypoints_len/4

            for i in range(int(n)):
                tran_boxes.append(change_points[0 + i*4: 4 + i*4])

            tran_boxes = np.array(tran_boxes).reshape(-1,2)                       
            tran_x = []
            tran_y = []
            for k in range(len(tran_boxes)):
                tran_x.append(int(tran_boxes[k][0]))
                tran_y.append(int(tran_boxes[k][1]))
            max_x = max(tran_x)
            min_x = min(tran_x)
            max_y = max(tran_y)
            min_y = min(tran_x)
            ctr_x = new_W / 2
            ctr_y = new_H / 2
            origin_xmin = ctr_x - width / 2
            origin_xmax = ctr_x + width / 2
            origin_ymin = ctr_y - height / 2
            origin_ymax = ctr_y + height / 2
            cut_xmax = origin_xmax
            cut_xmin = origin_xmin
            cut_ymax = origin_ymax
            cut_ymin = origin_ymin
            if max_x >= origin_xmax:
                cut_xmax = max_x
            if min_x <= origin_xmin:
                cut_xmin = min_x
            if max_y >= origin_ymax:
                cut_ymax = max_y
            if min_y <= origin_ymin:
                cut_ymin = min_y
            for i in range(len(tran_boxes)):
                tran_x[i] = tran_x[i] - cut_xmin 
                tran_y[i] = tran_y[i] - cut_ymin 
            image = image.crop((cut_xmin,cut_ymin,cut_xmax,cut_ymax))
            tran_x = np.array(tran_x)
            tran_y = np.array(tran_y)

            boxes = []
            masks = []
            mty = []
            kes = []
            #GET FORMAT OF BOXES,MASKS
            for idx in range(int(tran_x.size/4)):
                x_points = [tran_x[4 * idx], tran_x[4*idx+1],tran_x[4*idx+2],tran_x[4*idx+3]]
                y_points = [tran_y[4 * idx], tran_y[4*idx+1],tran_y[4*idx+2],tran_y[4*idx+3]]

                l1 = LineString([(x_points[0], y_points[0]), (x_points[2], y_points[2])])
                l2 = LineString([(x_points[1], y_points[1]), (x_points[3], y_points[3])])
                p_l1l2 = l1.intersection(l2)
                poly1 = Polygon([(x_points[0], y_points[0]), (x_points[1], y_points[1]),
                                (x_points[2], y_points[2]), (x_points[3], y_points[3])])
                if not poly1.is_valid:
                    continue
                if not p_l1l2.within(poly1):
                    continue
                if poly1.area <= 10:
                    continue
                x_min = min(x_points)
                x_max = max(x_points)
                y_min = min(y_points)
                y_max = max(y_points)
                width = max(0, x_max - x_min + 1)
                height = max(0, y_max - y_min + 1)
                if width == 0 or height == 0:
                    continue
                boxes.append([x_min,y_min,width,height])

                #get mask format
                one_point = [[tran_x[4*idx],tran_y[4*idx],tran_x[4*idx+1],tran_y[4*idx+1],tran_x[4*idx+2],tran_y[4*idx+2],tran_x[4*idx+3],tran_y[4*idx+3]]]
                masks.append(one_point)

                #get matchtype format
                mean_x = np.mean(x_points)
                mean_y = np.mean(y_points)
                xt_sort = np.sort(x_points)
                yt_sort = np.sort(y_points)
                xt_argsort = list(np.argsort(x_points))
                yt_argsort = list(np.argsort(y_points))
                ldx = []
                for ildx in range(4):
                    ldx.append(yt_argsort.index(xt_argsort[ildx]))
                all_types = [[1,2,3,4],[1,2,4,3],[1,3,2,4],[1,3,4,2],[1,4,2,3],[1,4,3,2],\
                                [2,1,3,4],[2,1,4,3],[2,3,1,4],[2,3,4,1],[2,4,1,3],[2,4,3,1],\
                                [3,1,2,4],[3,1,4,2],[3,2,1,4],[3,2,4,1],[3,4,1,2],[3,4,2,1],\
                                [4,1,2,3],[4,1,3,2],[4,2,1,3],[4,2,3,1],[4,3,1,2],[4,3,2,1]]
                all_types = [[all_types[iat][0]-1,all_types[iat][1]-1,all_types[iat][2]-1,all_types[iat][3]-1] for iat in range(24)]
                match_type = all_types.index(ldx)
                mty.append(match_type)

                half_x = (xt_sort + mean_x) / 2
                half_y = (yt_sort + mean_y) / 2

                keypoints = []
                keypoints.append(mean_x)
                keypoints.append(mean_y)
                keypoints.append(2)
                for i in range(4):
                    keypoints.append(half_x[i])
                    keypoints.append(mean_y)
                    keypoints.append(2)
                for i in range(4):
                    keypoints.append(mean_x)
                    keypoints.append(half_y[i])
                    keypoints.append(2)
                try:
                    keypoints.append(int(p_l1l2.x))
                    keypoints.append(int(p_l1l2.y))
                    keypoints.append(2)
                except Exception as e:
                    continue
                kes.append(keypoints)
            #IF ENCOUNTER THAT NO BOX IN A TRANSFORMED IMAGE, RETURN PRIMARY IMAGE AND TARGET
            if kes == []:
                image = image1
                target = target1
                return image,target
            classes = []
            for i in range(len(boxes)):
                classes.append(1)
            classes = torch.tensor(classes)
            #GET NEW TARGET
            boxes = torch.as_tensor(boxes).reshape(-1, 4)  
            target = BoxList(boxes, image.size, mode="xywh").convert("xyxy")

            target.add_field("labels",classes)

            masks = SegmentationMask(masks, image.size)
            target.add_field("masks", masks)

        return image,target