scripts/data_converter/nuplan/utils/canvas_3d.py

"""
Written by Jinhyung Park

Simple 3D visualization for 3D points & boxes. Intended as a simple, hackable
alternative to mayavi for certain point cloud tasks.
"""

import numpy as np
import cv2
import copy
from functools import partial
import matplotlib


class Canvas_3D(object):
    def __init__(
        self,
        canvas_shape=(500, 1000),
        camera_center_coords=(20, -40, 15),     
        camera_focus_coords=(-4 + 0.9396926, 0, 4 - 0.34202014),
        focal_length=None,
        canvas_bg_color=(0, 0, 0),
    ):
        """
        Args:
            canvas_shape (Tuple[Int]): Canvas image size - height & width.
            camera_center_coords (Tuple[Float]): Location of camera center in
                3D space. x -> right, y -> front, z -> height
            camera_focus_coords (Tuple[Float]): Intuitively, what point in 3D
                space is the camera pointed at? These are absolute coordinates,
                *not* relative to camera center.
            focal_length (None | Int):
                None: Half of the max of height & width of canvas_shape. This
                    seems to be a decent default.
                Int: Specified directly.
            canvas_bg_color (Tuple[Int]): RGB (0 ~ 255) of canvas background
                color.
        """

        self.canvas_shape = canvas_shape
        self.H, self.W = self.canvas_shape
        self.canvas_bg_color = canvas_bg_color

        self.camera_center_coords = camera_center_coords
        self.camera_focus_coords = camera_focus_coords

        if focal_length is None:
            self.focal_length = max(self.H, self.W) // 2
        else:
            self.focal_length = focal_length

        # Setup extrinsics and intrinsics of this virtual camera.
        self.ext_matrix = self.get_extrinsic_matrix(
            self.camera_center_coords, self.camera_focus_coords
        )
        self.int_matrix = np.array(
            [
                [self.focal_length, 0, self.W // 2, 0],
                [0, self.focal_length, self.H // 2, 0],
                [0, 0, 1, 0],
            ]
        )

        self.clear_canvas()

    def get_canvas(self):
        return self.canvas

    def clear_canvas(self):
        self.canvas = np.zeros((self.H, self.W, 3), dtype=np.uint8)
        self.canvas[..., :] = self.canvas_bg_color

    def get_canvas_coords(self, xyz, depth_min=0.1, return_depth=False):
        """
        Projects XYZ points onto the canvas and returns the projected canvas
        coordinates.

        Args:
            xyz (ndarray): (N, 3+) array of coordinates. Additional columns
                beyond the first three are ignored.
            depth_min (Float): Only points with a projected depth larger
                than this value are "valid".
            return_depth (Boolean): Whether to additionally return depth of
                projected points.
        Returns:
            canvas_xy (ndarray): (N, 2) array of projected canvas coordinates.
                "x" is dim0, "y" is dim1 of canvas.
            valid_mask (ndarray): (N,) boolean mask indicating which of
                canvas_xy fits into canvas (are visible from virtual camera).
            depth (ndarray): Optionally returned (N,) array of depth values
        """
        xyz = np.copy(xyz)  # prevent in-place modifications
        xyz = xyz[:, :3]

        xyz_hom = np.concatenate(
            [xyz, np.ones((xyz.shape[0], 1), dtype=np.float32)], axis=1
        )
        img_pts = (self.int_matrix @ self.ext_matrix @ xyz_hom.T).T

        depth = img_pts[:, 2]
        xy = img_pts[:, :2] / depth[:, None]
        xy_int = xy.round().astype(np.int32)

        # Flip X and Y so "x" is dim0, "y" is dim1 of canvas
        xy_int = xy_int[:, ::-1]

        valid_mask = (
            (depth > depth_min)
            & (xy_int[:, 0] >= 0)
            & (xy_int[:, 0] < self.H)
            & (xy_int[:, 1] >= 0)
            & (xy_int[:, 1] < self.W)
        )

        if return_depth:
            return xy_int, valid_mask, depth
        else:
            return xy_int, valid_mask

    def draw_canvas_points(
        self, canvas_xy, radius=-1, colors=None, colors_operand=None
    ):
        """
        Draws canvas_xy onto self.canvas.

        Args:
            canvas_xy (ndarray): (N, 2) array of *valid* canvas coordinates.
                "x" is dim0, "y" is dim1 of canvas.
            radius (Int):
                -1: Each point is visualized as a single pixel.
                r: Each point is visualized as a circle with radius r.
            colors:
                None: colors all points white.
                Tuple: RGB (0 ~ 255), indicating a single color for all points.
                ndarray: (N, 3) array of RGB values for each point.
                String: Such as "Spectral", uses a matplotlib cmap, with the
                    operand (the value cmap is called on for each point) being
                    colors_operand.
            colors_operand (ndarray): (N,) array of values cooresponding to
                canvas_xy, to be used only if colors is a cmap. Unlike
                Canvas_BEV, cannot be None if colors is a String.
        """
        if len(canvas_xy) == 0:
            return

        if colors is None:
            colors = np.full((len(canvas_xy), 3), fill_value=255, dtype=np.uint8)
        elif isinstance(colors, tuple):
            assert len(colors) == 3
            colors_tmp = np.zeros((len(canvas_xy), 3), dtype=np.uint8)
            colors_tmp[..., : len(colors)] = np.array(colors)
            colors = colors_tmp
        elif isinstance(colors, np.ndarray):
            assert len(colors) == len(canvas_xy)
            colors = colors.astype(np.uint8)
        elif isinstance(colors, str):
            assert colors_operand is not None
            colors = matplotlib.cm.get_cmap(colors)

            # Normalize 0 ~ 1 for cmap
            colors_operand = colors_operand - colors_operand.min()
            colors_operand = colors_operand / colors_operand.max()

            # Get cmap colors - note that cmap returns (*input_shape, 4), with
            # colors scaled 0 ~ 1
            colors = (colors(colors_operand)[:, :3] * 255).astype(np.uint8)
        else:
            raise Exception(
                "colors type {} was not an expected type".format(type(colors))
            )

        if radius == -1:
            self.canvas[canvas_xy[:, 0], canvas_xy[:, 1], :] = colors
        else:
            for color, (x, y) in zip(colors.tolist(), canvas_xy.tolist()):
                self.canvas = cv2.circle(
                    self.canvas, (y, x), radius, color, -1, lineType=cv2.LINE_AA
                )

    def draw_lines(self, start_xyz, end_xyz, colors=(255, 255, 255), thickness=1):
        """
        Draws lines between provided 3D points.

        Args:
            start_xyz (ndarray): Shape (N, 3) of 3D points to start from.
            end_xyz (ndarray): Shape (N, 3) of 3D points to end at. Same length
                as start_xyz.
            colors:
                None: colors all points white.
                Tuple: RGB (0 ~ 255), indicating a single color for all points.
                ndarray: (N, 3) array of RGB values for each point.
            thickness (Int):
                Thickness of drawn cv2 line.
        """
        if colors is None:
            colors = np.full((len(canvas_xy), 3), fill_value=255, dtype=np.uint8)
        elif isinstance(colors, tuple):
            assert len(colors) == 3
            colors_tmp = np.zeros((len(canvas_xy), 3), dtype=np.uint8)
            colors_tmp[..., : len(colors)] = np.array(colors)
            colors = colors_tmp
        elif isinstance(colors, np.ndarray):
            assert len(colors) == len(canvas_xy)
            colors = colors.astype(np.uint8)
        else:
            raise Exception(
                "colors type {} was not an expected type".format(type(colors))
            )

        start_pts_xy, start_pts_valid_mask, start_pts_d = self.get_canvas_coords(
            start_xyz, True
        )
        end_pts_xy, end_pts_valid_mask, end_pts_d = self.get_canvas_coords(
            end_xyz, True
        )

        for idx, (color, start_pt_xy, end_pt_xy) in enumerate(
            zip(colors.tolist(), start_pts_xy.tolist(), end_pts_xy.tolist())
        ):

            if start_pts_valid_mask[idx] and end_pts_valid_mask[idx]:
                self.canvas = cv2.line(
                    self.canvas,
                    tuple(start_pt_xy[::-1]),
                    tuple(end_pt_xy[::-1]),
                    color=color,
                    thickness=thickness,
                    lineType=cv2.LINE_AA,
                )

    def draw_boxes(
        self,
        boxes=None,
        corners=None,
        colors=None,
        texts=None,
        depth_min=0.1,
        draw_incomplete_boxes=True,
        box_line_thickness=2,
        box_text_size=0.5,
        text_corner=1,
    ):
        """
        Draws 3D boxes.

        Args:
            boxes (ndarray): Shape (N, 7), each row representing a box of
                format (x, y, z, x_size, y_size, z_size, yaw). This function
                assumes *bottom center* - the xyz center of the provided box
                is the center of the bottom face of the 3D box, not the
                floating true center of the 3D box.
            colors:
                None: colors all points white.
                Tuple: RGB (0 ~ 255), indicating a single color for all points.
                ndarray: (N, 3) array of RGB values for each point.
            texts (List[String]): Length N; text to write next to boxes.
            depth_min (Float): Only box corners with a projected depth larger
                than this value are drawn if draw_incomplete_boxes is True.
            draw_incomplete_boxes (Boolean): If any boxes are incomplete,
                meaning it has a corner out of view based on depth_min, decide
                whether to draw them at all.
            thickness (Int):
                Thickness of drawn cv2 box lines.
            box_line_thickness (int): cv2 line/text thickness
            box_text_size (float): cv2 putText size
            text_corner (int): 0 ~ 7. Which corner of 3D box to write text at.
        """

        num_boxes = len(boxes) if boxes is not None else len(corners)

        # Setup colors
        if colors is None:
            colors = np.full((num_boxes, 3), fill_value=255, dtype=np.uint8)
        elif isinstance(colors, tuple):
            assert len(colors) == 3
            colors_tmp = np.zeros((num_boxes, 3), dtype=np.uint8)
            colors_tmp[..., : len(colors)] = np.array(colors)
            colors = colors_tmp
        elif isinstance(colors, np.ndarray):
            assert len(colors) == num_boxes
            colors = colors.astype(np.uint8)
        else:
            raise Exception(
                "colors type {} was not an expected type".format(type(colors))
            )

        if boxes is not None:
            # boxes is N x 7
            boxes = np.copy(boxes)  # prevent in-place modifications
            assert len(boxes.shape) == 2

            dims = boxes[:, 3:6]
            corners_norm = np.stack(np.unravel_index(np.arange(8), [2] * 3), axis=1)

            corners_norm = corners_norm[[0, 1, 3, 2, 4, 5, 7, 6]]
            # use relative origin [0.5, 0.5, 0], assuming bottom center
            corners_norm = corners_norm - np.array([0.5, 0.5, 0])
            corners = dims.reshape(-1, 1, 3) * corners_norm.reshape([1, 8, 3])
            # rotate around z axis
            angles = boxes[:, 6]
            rot_sin = np.sin(angles)
            rot_cos = np.cos(angles)
            ones = np.ones_like(rot_cos)
            zeros = np.zeros_like(rot_cos)
            rot_mat_T = np.stack(
                [
                    np.stack([rot_cos, -rot_sin, zeros]),
                    np.stack([rot_sin, rot_cos, zeros]),
                    np.stack([zeros, zeros, ones]),
                ]
            )
            corners = np.einsum("aij,jka->aik", corners, rot_mat_T)
            corners += boxes[:, :3].reshape(-1, 1, 3)  # N x 8 x 3

        elif corners is not None:
            corners = corners

        # Now we have corners. Need them on the canvas 2D space.
        corners_xy, valid_mask = self.get_canvas_coords(
            corners.reshape(-1, 3), depth_min=depth_min
        )
        corners_xy = corners_xy.reshape(-1, 8, 2)
        valid_mask = valid_mask.reshape(-1, 8)

        # Now draw them with lines in correct places
        for i, (color, curr_corners_xy, curr_valid_mask) in enumerate(
            zip(colors.tolist(), corners_xy.tolist(), valid_mask.tolist())
        ):

            if not draw_incomplete_boxes and sum(curr_valid_mask) != 8:
                # Some corner is invalid, don't draw the box at all.
                continue

            for start, end in [
                (0, 1),
                (1, 2),
                (2, 3),
                (3, 0),
                (0, 4),
                (1, 5),
                (2, 6),
                (3, 7),
                (4, 5),
                (5, 6),
                (6, 7),
                (7, 4),
            ]:
                if not (curr_valid_mask[start] and curr_valid_mask[end]):
                    continue  # start or end is not valid

                self.canvas = cv2.line(
                    self.canvas,
                    (curr_corners_xy[start][1], curr_corners_xy[start][0]),
                    (curr_corners_xy[end][1], curr_corners_xy[end][0]),
                    color=color,
                    thickness=box_line_thickness,
                    lineType=cv2.LINE_AA,
                )

            # If even a single line was drawn, add text as well.
            if sum(curr_valid_mask) > 0:
                if texts is not None:
                    self.canvas = cv2.putText(
                        self.canvas,
                        str(texts[i]),
                        (
                            curr_corners_xy[text_corner][1],
                            curr_corners_xy[text_corner][0],
                        ),
                        cv2.FONT_HERSHEY_SIMPLEX,
                        box_text_size,
                        color,
                        thickness=box_line_thickness,
                    )

    @staticmethod
    def cart2sph(xyz):
        x, y, z = xyz[:, 0], xyz[:, 1], xyz[:, 2]

        depth = np.linalg.norm(xyz, 2, axis=1)
        az = -np.arctan2(y, x)
        el = np.arcsin(z / depth)
        return az, el, depth

    @staticmethod
    def get_extrinsic_matrix(
        camera_center_coords,
        camera_focus_coords,
    ):
        """
        Args:
            camera_center_coords: (x, y, z) of where camera should be located
                in 3D space.
            camera_focus_coords: (x, y, z) of where camera should look at from
                camera_center_coords

        Thoughts:
            Remember that in camera coordiantes, pos x is right, pos y is up,
                pos z is forward.
        """
        center_x, center_y, center_z = camera_center_coords
        focus_x, focus_y, focus_z = camera_focus_coords
        az, el, depth = Canvas_3D.cart2sph(
            np.array([[focus_x - center_x, focus_y - center_y, focus_z - center_z]])
        )
        az = float(az)
        el = float(el)
        depth = float(depth)

        ### First, construct extrinsics
        ## Rotation matrix

        z_rot = np.array(
            [[np.cos(az), -np.sin(az), 0], [np.sin(az), np.cos(az), 0], [0, 0, 1]]
        )

        # el is rotation around y axis.
        y_rot = np.array(
            [
                [np.cos(-el), 0, -np.sin(-el)],
                [0, 1, 0],
                [np.sin(-el), 0, np.cos(-el)],
            ]
        )

        ## Now, how the z_rot and y_rot work (spherical coordiantes), is it
        ## computes rotations starting from the positive x axis and rotates
        ## positive x axis to the desired direction. The desired direction is
        ## the "looking direction" of the camera, which should actually be the
        ## z-axis. So should convert the points so that the x axis is the new z
        ## axis, and after the transformations.
        ## Why x -> z for points? If we think about rotating the camera, z
        ## should become x, so reverse when moving points.
        last_rot = np.array([[0, -1, 0], [0, 0, -1], [1, 0, 0]])  # x -> z

        # Put them together. Order matters. Make it hom.
        rot_matrix = np.eye(4, dtype=np.float32)
        rot_matrix[:3, :3] = last_rot @ y_rot @ z_rot

        ## Translation matrix
        trans_matrix = np.array(
            [
                [1, 0, 0, -center_x],
                [0, 1, 0, -center_y],
                [0, 0, 1, -center_z],
                [0, 0, 0, 1],
            ]
        )

        ## Finally, extrinsics matrix. Order matters - do trans then rot
        ext_matrix = rot_matrix @ trans_matrix

        return ext_matrix