generation/control/eval_landmark.py

import os
from glob import glob
from tqdm import tqdm
import numpy as np
from pathlib import Path
from skimage.io import imread, imsave
import cv2
import json

end_list = np.array([17, 22, 27, 42, 48, 31, 36, 68], dtype = np.int32) - 1
def plot_kpts(image, kpts, color = 'g'):
    ''' Draw 68 key points
    Args: 
        image: the input image
        kpt: (68, 3).
    '''
    if color == 'r':
        c = (255, 0, 0)
    elif color == 'g':
        c = (0, 255, 0)
    elif color == 'b':
        c = (255, 0, 0)
    image = image.copy()
    kpts = kpts.copy()
    radius = max(int(min(image.shape[0], image.shape[1])/200), 1)
    for i in range(kpts.shape[0]):
        st = kpts[i, :2]
        if kpts.shape[1]==4:
            if kpts[i, 3] > 0.5:
                c = (0, 255, 0)
            else:
                c = (0, 0, 255)
        image = cv2.circle(image,(int(st[0]), int(st[1])), radius, c, radius*2)  
        if i in end_list:
            continue
        ed = kpts[i + 1, :2]
        image = cv2.line(image, (int(st[0]), int(st[1])), (int(ed[0]), int(ed[1])), (255, 255, 255), radius)
    return image

def plot_points(image, kpts, color = 'w'):
    ''' Draw 68 key points
    Args: 
        image: the input image
        kpt: (n, 3).
    '''
    if color == 'r':
        c = (255, 0, 0)
    elif color == 'g':
        c = (0, 255, 0)
    elif color == 'b':
        c = (0, 0, 255)
    elif color == 'y':
        c = (0, 255, 255)
    elif color == 'w':
        c = (255, 255, 255)
    image = image.copy()
    kpts = kpts.copy()
    kpts = kpts.astype(np.int32)
    radius = max(int(min(image.shape[0], image.shape[1])/200), 1)
    for i in range(kpts.shape[0]):
        st = kpts[i, :2]
        image = cv2.circle(image,(int(st[0]), int(st[1])), radius, c, radius*2)  
    return image

def generate_landmark2d(inputpath, savepath, n=0, device='cuda:0', vis=False):
    print(f'generate 2d landmarks')
    os.makedirs(savepath, exist_ok=True)
    import face_alignment
    detect_model = face_alignment.FaceAlignment(face_alignment.LandmarksType._2D, device=device, flip_input=False)
    
    imagepath_list = glob(os.path.join(inputpath, '*_{}.png'.format(n)))
    imagepath_list = sorted(imagepath_list)
    for imagepath in tqdm(imagepath_list):
        name = Path(imagepath).stem

        image = imread(imagepath)[:,:,:3]
        out = detect_model.get_landmarks(image)
        if out is None:
            continue
        kpt = out[0].squeeze()
        np.savetxt(os.path.join(savepath, f'{name}.txt'), kpt)
        if vis:
            image = cv2.imread(imagepath)
            image_point = plot_kpts(image, kpt)
            # check
            cv2.imwrite(os.path.join(savepath, f'{name}_overlay.jpg'), image_point)
            # background = np.zeros_like(image)
            # cv2.imwrite(os.path.join(savepath, f'{name}_line.jpg'), plot_kpts(background, kpt))
            # cv2.imwrite(os.path.join(savepath, f'{name}_point.jpg'), plot_points(background, kpt))
        # exit()

def landmark_comparison(lmk_folder, gt_lmk_folder, n=0):
    print(f'calculate reprojection error')
    lmk_err = []
    gt_lmk_folder = './data/celebhq-text/celeba-hq-landmark2d'
    with open('./data/celebhq-text/prompt_val_blip_full.json', 'rt') as f: # fill50k, COCO
        for line in f:
            val_data = json.loads(line)
    # for i in tqdm(range(2000)):
    for i in tqdm(range(len(val_data))):
        # import ipdb; ipdb.set_trace()
        # line = val_data[n]
        line = val_data[i]

        img_name = line["image"][:-4]
        lmk1_path = os.path.join(gt_lmk_folder, f'{img_name}.txt')

        lmk1 = np.loadtxt(lmk1_path) / 2
        lmk2_path = os.path.join(lmk_folder, f'result_{i}_{n}.txt')
        if not os.path.exists(lmk2_path):
            print(f'{lmk2_path} not exist')
            continue
        lmk2 = np.loadtxt(lmk2_path)
        lmk_err.append(np.mean(np.linalg.norm(lmk1 - lmk2, axis=1)))
    print(np.mean(lmk_err))
    np.save(os.path.join(lmk_folder, 'lmk_err.npy'), lmk_err)

n = 0
epoch = 19
gt_lmk_folder = './data/celebhq-text/celeba-hq-landmark2d'
# input_folder = os.path.join('./data/image_log_opt_lora_CelebA_landmark_lr_5-6_pe_diff_mlp_r_4_cayley_4gpu/results', str(epoch))
input_folder = os.path.join('log/image_log_householder_none_ADE20K_segm_eps_7e-06_pe_diff_mlp_l_8_8gpu_2024-05-15-19-33-41-650524/results', str(epoch))
# input_folder = os.path.join('log/image_log_oft_CelebA_landmark_eps_0.001_pe_diff_mlp_r_4_8gpu_2024-03-21-19-07-34-175825/train_with_norm/results', str(epoch))
save_folder = os.path.join(input_folder, 'landmark')

generate_landmark2d(input_folder, save_folder, n, device='cuda:0', vis=False)
landmark_comparison(save_folder, gt_lmk_folder, n)