training/dataset/utils/face_blend.py

'''
Create face mask and face boundary mask according to face landmarks,
so as to supervize the activation of Conv layer.
'''

import os
import numpy as np
import cv2
import dlib
import random
import argparse
from tqdm import tqdm
import time
from skimage import transform as trans
# from color_transfer import color_transfer
from .warp import gen_warp_params, warp_by_params, warp_mask


def crop_img_bbox(img, bbox, res, scale=1.3):
    x, y, w, h = bbox
    left, right = x, x+w
    top, bottom = y, y+h

    H, W, C = img.shape
    cx, cy = (left+right)//2, (top+bottom)//2
    w, h = (right-left)//2, (bottom-top)//2

    x1 = max(0, int(cx-w*scale))
    x2 = min(W, int(cx+w*scale))
    y1 = max(0, int(cy-h*scale))
    y2 = min(H, int(cy+h*scale))

    roi = img[y1:y2, x1:x2]
    roi = cv2.resize(roi, (res, res))

    return roi


def get_mask_center(mask):
    l, t, w, h = cv2.boundingRect(mask[:, :, 0:1].astype(np.uint8))
    center = int(l+w/2), int(t+h/2)
    return center


def get_5_keypoint(shape):
    def get_point(idx):
        # return [shape.part(idx).x, shape.part(idx).y]
        return shape[idx]

    def center(pt1, pt2):
        return [(pt1[0]+pt2[0])//2, (pt1[1]+pt2[1])//2]

    leye = np.array(center(get_point(36), get_point(39)),
                    dtype=int).reshape(-1, 2)
    reye = np.array(center(get_point(45), get_point(42)),
                    dtype=int).reshape(-1, 2)
    nose = np.array(get_point(30), dtype=int).reshape(-1, 2)
    lmouth = np.array(get_point(48),
                      dtype=int).reshape(-1, 2)
    rmouth = np.array(get_point(54),
                      dtype=int).reshape(-1, 2)

    pts = np.concatenate([leye, reye, nose, lmouth, rmouth], axis=0)

    return pts


def get_boundary(mask):
    if len(mask.shape) == 3:
        mask = mask[:, :, 0]
    mask = cv2.GaussianBlur(mask, (3, 3), 0)
    boundary = mask / 255.
    boundary = 4*boundary*(1.-boundary)
    return boundary


# def get_boundary(mask):
#     if len(mask.shape) == 3:
#         mask = mask[:, :, 0]
#     mask = cv2.GaussianBlur(mask, (3, 3), 0)
#     mask = mask.astype(np.uint8)

#     # Dilation and Erosion to find the boundary
#     dilated = cv2.dilate(mask, None, iterations=1)
#     boundary = cv2.subtract(dilated, mask)

#     # normalize the boundary to have values between 0 and 1
#     boundary = boundary / 255.

#     return boundary



def blur_mask(mask):
    blur_k = 2*np.random.randint(1, 10)-1
    
    #kernel = np.ones((blur_k+1, blur_k+1), np.uint8)
    #mask = cv2.erode(mask, kernel)
    
    mask = cv2.GaussianBlur(mask, (blur_k, blur_k), 0)


    return mask


def random_deform(pt, tgt, scale=0.3):
    x1, y1 = pt
    x2, y2 = tgt

    x = x1+(x2-x1)*np.random.rand()*scale
    y = y1+(y2-y1)*np.random.rand()*scale
    #print('before:', pt, ' after:', [int(x), int(y)])
    return [int(x), int(y)]


def get_specific_mask(img, shape, mtype='mouth', random_side=False):
    if mtype == 'eyes':
        landmarks = shape[42:45] if random.choice([True, False]) else shape[36:39]

    elif mtype == 'nose':
        landmarks = shape[27:35]

    elif mtype == 'mouth':
        landmarks = shape[48:60]

    elif mtype == 'eyebrows':
        landmarks = shape[22:26] if random.choice([True, False]) else shape[17:21]

    else:
        raise ValueError(f"Invalid mtype. Choose from 'eyes', 'nose', 'mouth', or 'eyebrows', but got {mtype}")

    # find convex hull
    hull = cv2.convexHull(landmarks)
    hull = hull.astype(int)

    # mask
    hull_mask = np.zeros_like(img)
    cv2.fillPoly(hull_mask, [hull], (255, 255, 255))
    mask = hull_mask
    return mask


def get_hull_mask(img, shape, mtype='hull'):
    if mtype == 'normal-hull':
        landmarks = np.array(shape)

        # find convex hull
        hull = cv2.convexHull(landmarks)
        hull = hull.astype(int)

        # full face mask
        hull_mask = np.zeros_like(img)
        cv2.fillPoly(hull_mask, [hull], (255, 255, 255))
        mask = hull_mask

    elif mtype == 'inner-hull':
        landmarks = shape[17:]
        landmarks = np.array(landmarks)

        # find convex hull
        hull = cv2.convexHull(landmarks)
        hull = hull.astype(int)

        # full face mask
        hull_mask = np.zeros_like(img)
        cv2.fillPoly(hull_mask, [hull], (255, 255, 255))

        mask = hull_mask

    elif mtype == 'inner-hull-no-eyebrow':
        landmarks = shape[27:]
        landmarks = np.array(landmarks)
        # find convex hull
        hull = cv2.convexHull(landmarks)
        hull = hull.astype(int)

        # full face mask
        hull_mask = np.zeros_like(img)
        cv2.fillPoly(hull_mask, [hull], (255, 255, 255))

        mask = hull_mask

    elif mtype == 'mouth-hull':
        landmarks = shape[2:15]
        #landmarks.append(shape[29])
        landmarks = np.concatenate([landmarks, shape[29].reshape(1, -1)], axis=0)

        # find convex hull
        hull = cv2.convexHull(landmarks)
        hull = hull.astype(int)

        # full face mask
        hull_mask = np.zeros_like(img)
        cv2.fillPoly(hull_mask, [hull], (255, 255, 255))

        # kernel = np.ones((2, 2), np.uint8)
        # c_mask = cv2.dilate(hull_mask, kernel, iterations=1)
        mask = hull_mask

    elif mtype == 'whole-hull':
        face_height = shape[9][1] - shape[22][1]
        landmarks = []
        for i in range(27):
            lmk = shape[i]
            if i >= 5 and i <= 11:
                x, y = lmk[0], lmk[1]
                lmk = [x, max(0, y+15)]
            # lift the eyebrows to get a larger landmark convex hull
            if i >= 18 and i <= 27:
                x, y = lmk[0], lmk[1]
                lmk = [x, max(0, y-face_height//4)]

            landmarks.append(lmk)

        # find convex hull
        landmarks = np.array(landmarks, dtype=np.int32)
        hull = cv2.convexHull(landmarks)
        hull = np.reshape(hull, (1, -1, 2))

        # full face mask
        hull_mask = np.zeros_like(img)
        cv2.fillPoly(hull_mask, [hull], (255, 255, 255))

        # kernel = np.ones((2, 2), np.uint8)
        # c_mask = cv2.dilate(hull_mask, kernel, iterations=1)
        mask = hull_mask
    '''
    elif mtype == 'rect':
        cnt = []
        for idx in [5, 11, 17, 26]:
            cnt.append(shape[idx])
        x, y, w, h = cv2.boundingRect(np.array(cnt))
        rect_mask = np.zeros_like(img)
        cv2.rectangle(rect_mask, (x, y), (x+w, y+h),
                      (255, 255, 255), cv2.FILLED)
        mask = rect_mask
    '''
    return mask


def get_mask(shape, img, std=20, deform=True, restrict_mask=None):
    mask_type = [
        'normal-hull', 
        'inner-hull', 
        'inner-hull-no-eyebrow', 
        'mouth-hull', 
        'whole-hull'
    ]
    max_mask = get_hull_mask(img, shape, 'whole-hull')
    if deform:
        mtype = mask_type[np.random.randint(len(mask_type))]
        if mtype == 'rect':
            mask = get_hull_mask(img, shape, 'inner-hull-no-eyebrow')
            x, y, w, h = cv2.boundingRect(mask[:,:,0])
            for i in range(y, y+h):
                for j in range(x, x+w):
                    for k in range(mask.shape[2]):
                        mask[i, j, k] = 255
        else:
            mask = get_hull_mask(img, shape, mtype)

        # random deform
        if np.random.rand() < 0.9:
            mask = warp_mask(mask, std=std)

        # # random erode/dilate
        # prob = np.random.rand()
        # if prob < 0.3:
        #     erode_k = 2*np.random.randint(1, 10)+1
        #     kernel = np.ones((erode_k, erode_k), np.uint8)
        #     mask = cv2.erode(mask, kernel)
        # elif prob < 0.6:
        #     erode_k = 2*np.random.randint(1, 10)+1
        #     kernel = np.ones((erode_k, erode_k), np.uint8)
        #     mask = cv2.dilate(mask, kernel)
    else:
        mask = max_mask.copy()
    
    if restrict_mask is not None:
        mask = mask*(restrict_mask//255)

    # restrict mask range
    mask = mask *(max_mask//255)

    # random blur
    if deform and np.random.rand() < 0.9:
        mask = blur_mask(mask)

    return mask[:,:,0]

def mask_postprocess(mask):
    # random erode/dilate
    prob = np.random.rand()
    if prob < 0.3:
        erode_k = 2*np.random.randint(1, 10)+1
        kernel = np.ones((erode_k, erode_k), np.uint8)
        mask = cv2.erode(mask, kernel)
    elif prob < 0.6:
        erode_k = 2*np.random.randint(1, 10)+1
        kernel = np.ones((erode_k, erode_k), np.uint8)
        mask = cv2.dilate(mask, kernel)
    
    # random blur
    if np.random.rand() < 0.9:
        mask = blur_mask(mask)

    return mask


def get_affine_param(from_, to_):
        # use skimage tranformation
    tform = trans.SimilarityTransform()
    tform.estimate(from_.astype(np.float32), to_.astype(
        np.float32))  # tform.estimate(from_, to_)
    M = tform.params[0:2, :]

    return M


def random_sharpen_img(img):
    cand = ['bsharpen', 'gsharpen']  # , 'none']
    mode = cand[np.random.randint(len(cand))]
    # print('sharpen mode:', mode)
    if mode == "bsharpen":
        # Sharpening using filter2D
        kernel = np.ones((3, 3)) * (-1)
        kernel[1, 1] = 9
        #kernel /= 9.
        out = cv2.filter2D(img, -1, kernel)
    elif mode == "gsharpen":
        # Sharpening using Weighted Method
        gaussain_blur = cv2.GaussianBlur(img, (0, 0), 3.0)
        out = cv2.addWeighted(
            img, 1.5, gaussain_blur, -0.5, 0, img)
    else:
        out = img

    return out


def random_blur_img(img):
    cand = ['avg', 'gaussion', 'med']  # , 'none']
    mode = cand[np.random.randint(len(cand))]
    # print('blur mode:', mode)
    ksize = 2*np.random.randint(1, 5)+1

    if mode == 'avg':
        # Averaging
        out = cv2.blur(img, (ksize, ksize))
    elif mode == 'gaussion':
        # Gaussian Blurring
        out = cv2.GaussianBlur(img, (ksize, ksize), 0)
    elif mode == 'med':
        # Median blurring
        out = cv2.medianBlur(img, ksize)
    else:
        out = img
    # elif mode == 'bilateral'
    #     # Bilateral Filtering
    #     out = cv2.bilateralFilter(img,9,75,75)

    return out


def random_warp_img(img, prob=0.5):
    H, W, C = img.shape
    param = gen_warp_params(W, flip=False)
    choice = [True, False]

    out = warp_by_params(param, img,
                         can_flip=False,  # choice[np.random.randint(2)],
                         can_transform=False,  # choice[np.random.randint(2)],
                         can_warp=(np.random.randint(10) < int(prob*10)),
                         border_replicate=choice[np.random.randint(2)])
    return out


def main(args):
    np.random.seed(int(time.time()))
    detector = dlib.get_frontal_face_detector()
    landmark_predictor = dlib.shape_predictor(args.model)

    src_im = cv2.imread(args.src)
    tgt_im = cv2.imread(args.tgt)

    H, W, C = tgt_im.shape
    src_im = cv2.resize(src_im, (W, H))

    def get_shape(img):
        img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        dets = detector(img, 1)
        det = dets[0]
        shape = landmark_predictor(img, det)

        return shape, det

    src_shape, src_det = get_shape(src_im)
    src_5_pts = get_5_keypoint(src_shape)
    src_mask = get_mask(src_shape, src_im, whole=True, deform=False)

    tgt_shape, tgt_det = get_shape(tgt_im)
    tgt_5_pts = get_5_keypoint(tgt_shape)
    tgt_mask = get_mask(tgt_shape, tgt_im, whole=False, deform=True)

    #aff_param = get_affine_param(src_5_pts, tgt_5_pts)

    # color transfer:
    mask = src_mask[:, :, 0:1]/255.
    ct_modes = ['lct', 'rct', 'idt', 'idt-m', 'mkl', 'mkl-m',
                'sot', 'sot-m', 'mix-m']  # , 'seamless-hist-match']
    for mode in ct_modes:
        colored_src = color_transfer(mode, src_im, tgt_im, mask)
        cv2.imwrite('{}_colored.png'.format(mode), colored_src)
    src_im = colored_src

    w1, h1 = src_det.right()-src_det.left(), src_det.bottom()-src_det.top()
    w2, h2 = tgt_det.right()-tgt_det.left(), tgt_det.bottom()-tgt_det.top()
    w_scale, h_scale = w2/w1, h2/h1

    scaled_src = cv2.resize(src_im, (int(W*w_scale), int(H*h_scale)))
    scaled_mask = cv2.resize(src_mask, (int(W*w_scale), int(H*h_scale)))

    src_5_pts[:, 0] = src_5_pts[:, 0]*w_scale
    src_5_pts[:, 1] = src_5_pts[:, 1]*h_scale
    aff_param = get_affine_param(src_5_pts, tgt_5_pts)

    aligned_src = cv2.warpAffine(
        scaled_src, aff_param, (W, H), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REFLECT)
    aligned_mask = cv2.warpAffine(
        scaled_mask, aff_param, (W, H), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REFLECT)

    center = get_mask_center(aligned_mask)
    print('mask center:', center)
    # colored_src = transfer_color(aligned_src, tgt_im)

    init_blend = cv2.seamlessClone(
        aligned_src, tgt_im, aligned_mask, center, cv2.NORMAL_CLONE)
    cv2.imwrite('init_blended.png', init_blend)

    # aligned_blend = cv2.warpAffine(
    #    colored_blend, aff_param, (W, H), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REFLECT)
    b_mask = tgt_mask[:, :, 0:1]/255.
    out_blend = init_blend*b_mask + tgt_im*(1. - b_mask)
    cv2.imwrite('out_blended.png', out_blend)

    res = 256
    blend_crop = crop_img_bbox(out_blend, tgt_det, res, scale=1.5)
    mask_crop = crop_img_bbox(tgt_mask, tgt_det, res, scale=1.5)
    boundary = get_boundary(mask_crop)

    cv2.imwrite('crop_blend.png', blend_crop)
    cv2.imwrite('crop_mask.png', mask_crop)
    cv2.imwrite('crop_bound.png', boundary*255)


if __name__ == "__main__":
    p = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter
    )
    p.add_argument('-s', '--src', type=str,
                   help='src image')
    p.add_argument('-t', '--tgt', type=str,
                   help='tgt image')
    p.add_argument('--model', type=str, default='/data1/yuchen/download/face_landmark/shape_predictor_68_face_landmarks.dat',
                   help="path to downloaded detector")
    args = p.parse_args()
    print(args)

    main(args)