models/cmp/utils/data_utils.py

from PIL import Image, ImageOps
import scipy.ndimage as ndimage
import cv2
import random
import numpy as np
from scipy.ndimage.filters import maximum_filter
from scipy import signal
cv2.ocl.setUseOpenCL(False)

def get_edge(data, blur=False):
    if blur:
        data = cv2.GaussianBlur(data, (3, 3), 1.)
    sobel = np.array([[1,0,-1],[2,0,-2],[1,0,-1]]).astype(np.float32)
    ch_edges = []
    for k in range(data.shape[2]):
        edgex = signal.convolve2d(data[:,:,k], sobel, boundary='symm', mode='same')
        edgey = signal.convolve2d(data[:,:,k], sobel.T, boundary='symm', mode='same')
        ch_edges.append(np.sqrt(edgex**2 + edgey**2))
    return sum(ch_edges)

def get_max(score, bbox):
    u = max(0, bbox[0])
    d = min(score.shape[0], bbox[1])
    l = max(0, bbox[2])
    r = min(score.shape[1], bbox[3])
    return score[u:d,l:r].max()

def nms(score, ks):
    assert ks % 2 == 1
    ret_score = score.copy()
    maxpool = maximum_filter(score, footprint=np.ones((ks, ks)))
    ret_score[score < maxpool] = 0.
    return ret_score

def image_flow_crop(img1, img2, flow, crop_size, phase):
    assert len(crop_size) == 2
    pad_h = max(crop_size[0] - img1.height, 0)
    pad_w = max(crop_size[1] - img1.width, 0)
    pad_h_half = int(pad_h / 2)
    pad_w_half = int(pad_w / 2)
    if pad_h > 0 or pad_w > 0:
        flow_expand = np.zeros((img1.height + pad_h, img1.width + pad_w, 2), dtype=np.float32)
        flow_expand[pad_h_half:pad_h_half+img1.height, pad_w_half:pad_w_half+img1.width, :] = flow
        flow = flow_expand
        border = (pad_w_half, pad_h_half, pad_w - pad_w_half, pad_h - pad_h_half)
        img1 = ImageOps.expand(img1, border=border, fill=(0,0,0))
        img2 = ImageOps.expand(img2, border=border, fill=(0,0,0))
    if phase == 'train':
        hoff = int(np.random.rand() * (img1.height - crop_size[0]))
        woff = int(np.random.rand() * (img1.width - crop_size[1]))
    else:
        hoff = (img1.height - crop_size[0]) // 2
        woff = (img1.width - crop_size[1]) // 2

    img1 = img1.crop((woff, hoff, woff+crop_size[1], hoff+crop_size[0]))
    img2 = img2.crop((woff, hoff, woff+crop_size[1], hoff+crop_size[0]))
    flow = flow[hoff:hoff+crop_size[0], woff:woff+crop_size[1], :]
    offset = (hoff, woff)
    return img1, img2, flow, offset

def image_crop(img, crop_size):
    pad_h = max(crop_size[0] - img.height, 0)
    pad_w = max(crop_size[1] - img.width, 0)
    pad_h_half = int(pad_h / 2)
    pad_w_half = int(pad_w / 2)
    if pad_h > 0 or pad_w > 0:
        border = (pad_w_half, pad_h_half, pad_w - pad_w_half, pad_h - pad_h_half)
        img = ImageOps.expand(img, border=border, fill=(0,0,0))
    hoff = (img.height - crop_size[0]) // 2
    woff = (img.width - crop_size[1]) // 2
    return img.crop((woff, hoff, woff+crop_size[1], hoff+crop_size[0])), (pad_w_half, pad_h_half)

def image_flow_resize(img1, img2, flow, short_size=None, long_size=None):
    assert (short_size is None) ^ (long_size is None)
    w, h = img1.width, img1.height
    if short_size is not None:
        if w < h:
            neww = short_size
            newh = int(short_size / float(w) * h)
        else:
            neww = int(short_size / float(h) * w)
            newh = short_size
    else:
        if w < h:
            neww = int(long_size / float(h) * w)
            newh = long_size
        else:
            neww = long_size
            newh = int(long_size / float(w) * h)
    img1 = img1.resize((neww, newh), Image.BICUBIC)
    img2 = img2.resize((neww, newh), Image.BICUBIC)
    ratio = float(newh) / h
    flow = cv2.resize(flow.copy(), (neww, newh), interpolation=cv2.INTER_LINEAR) * ratio
    return img1, img2, flow, ratio

def image_resize(img, short_size=None, long_size=None):
    assert (short_size is None) ^ (long_size is None)
    w, h = img.width, img.height
    if short_size is not None:
        if w < h:
            neww = short_size
            newh = int(short_size / float(w) * h)
        else:
            neww = int(short_size / float(h) * w)
            newh = short_size
    else:
        if w < h:
            neww = int(long_size / float(h) * w)
            newh = long_size
        else:
            neww = long_size
            newh = int(long_size / float(w) * h)
    img = img.resize((neww, newh), Image.BICUBIC)
    return img, [w, h]


def image_pose_crop(img, posemap, crop_size, scale):
    assert len(crop_size) == 2
    assert crop_size[0] <= img.height
    assert crop_size[1] <= img.width
    hoff = (img.height - crop_size[0]) // 2
    woff = (img.width - crop_size[1]) // 2
    img = img.crop((woff, hoff, woff+crop_size[1], hoff+crop_size[0]))
    posemap = posemap[hoff//scale:hoff//scale+crop_size[0]//scale, woff//scale:woff//scale+crop_size[1]//scale,:]
    return img, posemap

def neighbor_elim(ph, pw, d):
    valid = np.ones((len(ph))).astype(np.int)
    h_dist = np.fabs(np.tile(ph[:,np.newaxis], [1,len(ph)]) - np.tile(ph.T[np.newaxis,:], [len(ph),1]))
    w_dist = np.fabs(np.tile(pw[:,np.newaxis], [1,len(pw)]) - np.tile(pw.T[np.newaxis,:], [len(pw),1]))
    idx1, idx2 = np.where((h_dist < d) & (w_dist < d))
    for i,j in zip(idx1, idx2):
        if valid[i] and valid[j] and i != j:
            if np.random.rand() > 0.5:
                valid[i] = 0
            else:
                valid[j] = 0
    valid_idx = np.where(valid==1)
    return ph[valid_idx], pw[valid_idx]

def remove_border(mask):
        mask[0,:] = 0
        mask[:,0] = 0
        mask[mask.shape[0]-1,:] = 0
        mask[:,mask.shape[1]-1] = 0

def flow_sampler(flow, strategy=['grid'], bg_ratio=1./6400, nms_ks=15, max_num_guide=-1, guidepoint=None):
    assert bg_ratio >= 0 and bg_ratio <= 1, "sampling ratio must be in (0, 1]"
    for s in strategy:
        assert s in ['grid', 'uniform', 'gradnms', 'watershed', 'single', 'full', 'specified'], "No such strategy: {}".format(s)
    h = flow.shape[0]
    w = flow.shape[1]
    ds = max(1, max(h, w) // 400) # reduce computation

    if 'full' in strategy:
        sparse = flow.copy()
        mask = np.ones(flow.shape, dtype=np.int)
        return sparse, mask

    pts_h = []
    pts_w = []
    if 'grid' in strategy:
        stride = int(np.sqrt(1./bg_ratio))
        mesh_start_h = int((h - h // stride * stride) / 2)
        mesh_start_w = int((w - w // stride * stride) / 2)
        mesh = np.meshgrid(np.arange(mesh_start_h, h, stride), np.arange(mesh_start_w, w, stride))
        pts_h.append(mesh[0].flat)
        pts_w.append(mesh[1].flat)
    if 'uniform' in strategy:
        pts_h.append(np.random.randint(0, h, int(bg_ratio * h * w)))
        pts_w.append(np.random.randint(0, w, int(bg_ratio * h * w)))
    if "gradnms" in strategy:
        ks = w // ds // 20
        edge = get_edge(flow[::ds,::ds,:])
        kernel = np.ones((ks, ks), dtype=np.float32) / (ks * ks)
        subkernel = np.ones((ks//2, ks//2), dtype=np.float32) / (ks//2 * ks//2)
        score = signal.convolve2d(edge, kernel, boundary='symm', mode='same')
        subscore = signal.convolve2d(edge, subkernel, boundary='symm', mode='same')
        score = score / score.max() - subscore / subscore.max()
        nms_res = nms(score, nms_ks)
        pth, ptw = np.where(nms_res > 0.1)
        pts_h.append(pth * ds)
        pts_w.append(ptw * ds)
    if "watershed" in strategy:
        edge = get_edge(flow[::ds,::ds,:])
        edge /= max(edge.max(), 0.01)
        edge = (edge > 0.1).astype(np.float32)
        watershed = ndimage.distance_transform_edt(1-edge)
        nms_res = nms(watershed, nms_ks)
        remove_border(nms_res)
        pth, ptw = np.where(nms_res > 0)
        pth, ptw = neighbor_elim(pth, ptw, (nms_ks-1)/2)
        pts_h.append(pth * ds)
        pts_w.append(ptw * ds)
    if "single" in strategy:
        pth, ptw = np.where((flow[:,:,0] != 0) | (flow[:,:,1] != 0))
        randidx = np.random.randint(len(pth))
        pts_h.append(pth[randidx:randidx+1])
        pts_w.append(ptw[randidx:randidx+1])
    if 'specified' in strategy:
        assert guidepoint is not None, "if using \"specified\", switch \"with_info\" on."
        pts_h.append(guidepoint[:,1])
        pts_w.append(guidepoint[:,0])

    pts_h = np.concatenate(pts_h)
    pts_w = np.concatenate(pts_w)

    if max_num_guide == -1:
        max_num_guide = np.inf

    randsel = np.random.permutation(len(pts_h))[:len(pts_h)]
    selidx = randsel[np.arange(min(max_num_guide, len(randsel)))]
    pts_h = pts_h[selidx]
    pts_w = pts_w[selidx]

    sparse = np.zeros(flow.shape, dtype=flow.dtype)
    mask = np.zeros(flow.shape, dtype=np.int)
    
    sparse[:, :, 0][(pts_h, pts_w)] = flow[:, :, 0][(pts_h, pts_w)]
    sparse[:, :, 1][(pts_h, pts_w)] = flow[:, :, 1][(pts_h, pts_w)]
    
    mask[:,:,0][(pts_h, pts_w)] = 1
    mask[:,:,1][(pts_h, pts_w)] = 1
    return sparse, mask

def image_flow_aug(img1, img2, flow, flip_horizon=True):
    if flip_horizon:
        if random.random() < 0.5:
            img1 = img1.transpose(Image.FLIP_LEFT_RIGHT)
            img2 = img2.transpose(Image.FLIP_LEFT_RIGHT)
            flow = flow[:,::-1,:].copy()
            flow[:,:,0] = -flow[:,:,0]
    return img1, img2, flow

def flow_aug(flow, reverse=True, scale=True, rotate=True):
    if reverse:
        if random.random() < 0.5:
            flow = -flow
    if scale:
        rand_scale = random.uniform(0.5, 2.0)
        flow = flow * rand_scale
    if rotate and random.random() < 0.5:
        lengh = np.sqrt(np.square(flow[:,:,0]) + np.square(flow[:,:,1]))
        alpha = np.arctan(flow[:,:,1] / flow[:,:,0])
        theta = random.uniform(0, np.pi*2)
        flow[:,:,0] = lengh * np.cos(alpha + theta)
        flow[:,:,1] = lengh * np.sin(alpha + theta)
    return flow

def draw_gaussian(img, pt, sigma, type='Gaussian'):
    # Check that any part of the gaussian is in-bounds
    ul = [int(pt[0] - 3 * sigma), int(pt[1] - 3 * sigma)]
    br = [int(pt[0] + 3 * sigma + 1), int(pt[1] + 3 * sigma + 1)]
    if (ul[0] >= img.shape[1] or ul[1] >= img.shape[0] or
            br[0] < 0 or br[1] < 0):
        # If not, just return the image as is
        return img

    # Generate gaussian
    size = 6 * sigma + 1
    x = np.arange(0, size, 1, float)
    y = x[:, np.newaxis]
    x0 = y0 = size // 2
    # The gaussian is not normalized, we want the center value to equal 1
    if type == 'Gaussian':
        g = np.exp(- ((x - x0) ** 2 + (y - y0) ** 2) / (2 * sigma ** 2))
    elif type == 'Cauchy':
        g = sigma / (((x - x0) ** 2 + (y - y0) ** 2 + sigma ** 2) ** 1.5)

    # Usable gaussian range
    g_x = max(0, -ul[0]), min(br[0], img.shape[1]) - ul[0]
    g_y = max(0, -ul[1]), min(br[1], img.shape[0]) - ul[1]
    # Image range
    img_x = max(0, ul[0]), min(br[0], img.shape[1])
    img_y = max(0, ul[1]), min(br[1], img.shape[0])

    img[img_y[0]:img_y[1], img_x[0]:img_x[1]] = g[g_y[0]:g_y[1], g_x[0]:g_x[1]]
    return img