lib/utils/vis_utils.py

import numpy as np
import torch
import cv2
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import os
import matplotlib.patches as patches
from sklearn.manifold import TSNE
import open3d as o3d

kintree = {
    'kintree': [[1, 0], [2, 1], [3, 2], [4, 3], [5, 1], [6, 5], [7, 6], [8, 1],
                [9, 8], [10, 9], [11, 10], [12, 8], [13, 12], [14, 13],
                [15, 0], [16, 0], [17, 15], [18, 16], [19, 14], [20, 19],
                [21, 14], [22, 11], [23, 22], [24, 11]],
    'color': [
        'k', 'r', 'r', 'r', 'b', 'b', 'b', 'k', 'r', 'r', 'r', 'b', 'b', 'b',
        'y', 'y', 'y', 'y', 'b', 'b', 'b', 'r', 'r', 'r'
    ]
}


def plotSkel3D(pts,
               config=kintree,
               ax=None,
               phi=0,
               theta=0,
               max_range=1,
               linewidth=4,
               color=None):
    multi = False
    if torch.is_tensor(pts):
        if len(pts.shape) == 3:
            print(">>> Visualize multiperson ...")
            multi = True
            if pts.shape[1] != 3:
                pts = pts.transpose(1, 2)
        elif len(pts.shape) == 2:
            if pts.shape[0] != 3:
                pts = pts.transpose(0, 1)
        else:
            raise RuntimeError('The dimension of the points is wrong!')
        pts = pts.detach().cpu().numpy()
    else:
        if pts.shape[0] != 3:
            pts = pts.T
    # pts : bn, 3, NumOfPoints or (3, N)
    if ax is None:
        print('>>> create figure ...')
        fig = plt.figure(figsize=[5, 5])
        ax = fig.add_subplot(111, projection='3d')
    for idx, (i, j) in enumerate(config['kintree']):
        if multi:
            for b in range(pts.shape[0]):
                ax.plot([pts[b][0][i], pts[b][0][j]],
                        [pts[b][1][i], pts[b][1][j]],
                        [pts[b][2][i], pts[b][2][j]],
                        lw=linewidth,
                        color=config['color'][idx] if color is None else color,
                        alpha=1)
        else:
            ax.plot([pts[0][i], pts[0][j]], [pts[1][i], pts[1][j]],
                    [pts[2][i], pts[2][j]],
                    lw=linewidth,
                    color=config['color'][idx],
                    alpha=1)
    if multi:
        for b in range(pts.shape[0]):
            ax.scatter(pts[b][0], pts[b][1], pts[b][2], color='r', alpha=1)
    else:
        ax.scatter(pts[0], pts[1], pts[2], color='r', alpha=1, s=0.5)
    ax.view_init(phi, theta)
    ax.set_xlim(-max_range, max_range)
    ax.set_ylim(-max_range, max_range)
    ax.set_zlim(-0.05, 2)

    # ax.axis('equal')
    plt.xlabel('x')
    plt.ylabel('y')
    # plt.zlabel('z')
    return ax


def plotSkel2D(pts,
               config=kintree,
               ax=None,
               linewidth=2,
               alpha=1,
               max_range=1,
               imgshape=None,
               thres=0.1):
    if len(pts.shape) == 2:
        pts = pts[None, :, :]  #(nP, nJ, 2/3)
    elif len(pts.shape) == 3:
        pass
    else:
        raise RuntimeError('The dimension of the points is wrong!')
    if torch.is_tensor(pts):
        pts = pts.detach().cpu().numpy()
    if pts.shape[2] == 3 or pts.shape[2] == 2:
        pts = pts.transpose((0, 2, 1))
    # pts : bn, 2/3, NumOfPoints or (2/3, N)
    if ax is None:
        fig = plt.figure(figsize=[5, 5])
        ax = fig.add_subplot(111)
    if 'color' in config.keys():
        colors = config['color']
    else:
        colors = ['b' for _ in range(len(config['kintree']))]

    def inrange(imgshape, pts):
        if pts[0] < 5 or \
           pts[0] > imgshape[1] - 5 or \
           pts[1] < 5 or \
           pts[1] > imgshape[0] - 5:
            return False
        else:
            return True

    for nP in range(pts.shape[0]):
        for idx, (i, j) in enumerate(config['kintree']):
            if pts.shape[1] == 3:  # with confidence
                if np.min(pts[nP][2][[i, j]]) < thres:
                    continue
                lw = linewidth * 2 * np.min(pts[nP][2][[i, j]])
            else:
                lw = linewidth
            if imgshape is not None:
                if inrange(imgshape, pts[nP, :, i]) and \
                    inrange(imgshape, pts[nP, :, j]):
                    pass
                else:
                    continue
            ax.plot([pts[nP][0][i], pts[nP][0][j]],
                    [pts[nP][1][i], pts[nP][1][j]],
                    lw=lw,
                    color=colors[idx],
                    alpha=1)
        # if pts.shape[1] > 2:
        # ax.scatter(pts[nP][0], pts[nP][1], s=10*(pts[nP][2]-thres), c='r')
    if False:
        ax.axis('equal')
        plt.xlabel('x')
        plt.ylabel('y')
    else:
        ax.axis('off')
    return ax


def draw_skeleton(img, kpts2d):
    cv_img = img.copy()
    for kp in kpts2d:
        if kp.shape[-1] == 2 or (kp.shape[-1] == 3 and kp[-1] > 0):
            cv_img = cv2.circle(cv_img, tuple(kp[:2].astype(int)), 10,
                                (255, 0, 0))
    return cv_img


def vis_frame(data_root, im_data, camera):
    from external.SMPL_CPP.build.python import pysmplceres
    from .smpl_renderer import Renderer

    imgs = [
        cv2.imread(os.path.join(data_root, im_path))
        for im_path in im_data['ims']
    ]
    imgs = [cv2.resize(img, (1024, 1024)) for img in imgs]

    Ks = np.array(camera['K'])
    Rs = np.array(camera['R'])
    Ts = np.array(camera['T']).transpose(0, 2, 1) / 1000

    faces = np.loadtxt('data/smpl/faces.txt').astype(np.int32)
    render = Renderer(height=1024, width=1024, faces=faces)
    vertices = pysmplceres.getVertices(im_data['smpl_result'])

    imgsrender = render.render_multiview(vertices[0], Ks, Rs, Ts, imgs)
    for img in imgsrender:
        plt.imshow(img[..., ::-1])
        plt.show()


def vis_skeleton_frame(data_root, im_data, camera):
    from external.SMPL_CPP.build.python import pysmplceres
    from .smpl_renderer import Renderer

    imgs = [
        cv2.imread(os.path.join(data_root, im_path))
        for im_path in im_data['ims']
    ]
    imgs = [cv2.resize(img, (1024, 1024)) for img in imgs]
    kpts2d = np.array(im_data['kpts2d'])

    for img, kpts in zip(imgs, kpts2d):
        _, ax = plt.subplots(1, 1)
        ax.imshow(img[..., ::-1])
        plotSkel2D(kpts, ax=ax)
        plt.show()


def vis_bbox(img, corners_2d, coord):
    _, ax = plt.subplots(1)
    ax.imshow(img)
    ax.add_patch(
        patches.Polygon(xy=corners_2d[[0, 1, 3, 2, 0, 4, 6, 2]],
                        fill=False,
                        linewidth=1,
                        edgecolor='g'))
    ax.add_patch(
        patches.Polygon(xy=corners_2d[[5, 4, 6, 7, 5, 1, 3, 7]],
                        fill=False,
                        linewidth=1,
                        edgecolor='g'))
    ax.plot(coord[:, 1], coord[:, 0], '.')
    plt.show()


def tsne_colors(data):
    """
    N x D np.array data
    """
    tsne = TSNE(n_components=1,
                verbose=1,
                perplexity=40,
                n_iter=300,
                random_state=0)
    tsne_results = tsne.fit_transform(data)
    tsne_results = np.squeeze(tsne_results)
    tsne_min = np.min(tsne_results)
    tsne_max = np.max(tsne_results)
    tsne_results = (tsne_results - tsne_min) / (tsne_max - tsne_min)
    colors = plt.cm.Spectral(tsne_results)[:, :3]
    return colors


def get_colored_pc(pts, rgb):
    pc = o3d.geometry.PointCloud()
    pc.points = o3d.utility.Vector3dVector(pts)
    colors = np.zeros_like(pts)
    colors += rgb
    pc.colors = o3d.utility.Vector3dVector(colors)
    return pc