scripts/data_converter/nuplan/utils/bev.py

"""
Written by Jinhyung Park

Simple BEV visualization for 3D points & boxes.
"""

import cv2
import matplotlib
import numpy as np

# np.set_printoptions(precision=3, suppress=True)


class Canvas_BEV(object):
    def __init__(
        self,
        canvas_shape=(2000, 2000),
        canvas_x_range=(-50.0, 50.0),
        canvas_y_range=(-50.0, 50.0),
        canvas_bg_color=(0, 0, 0),
    ):
        """
        Args:
            canvas_shape (Tuple[int]): Shape of BEV Canvas image. First element
                corresponds to X range, the second element to Y range.
            canvas_x_range (Tuple[int]): Range of X-coords to visualize. X is
                vertical: negative ~ positive is top ~ down.
            canvas_y_range (Tuple[int]): Range of Y-coords to visualize. Y is
                horizontal: negative ~ positive is left ~ right.
            canvas_bg_color (Tuple[int]): RGB (0 ~ 255) of Canvas background
                color.
        """

        # Sanity check ratios
        if (canvas_shape[0] / canvas_shape[1]) != (
            (canvas_x_range[0] - canvas_x_range[1])
            / (canvas_y_range[0] - canvas_y_range[1])
        ):

            print(
                "Not an error, but the x & y ranges are not "
                "proportional to canvas height & width."
            )

        self.canvas_shape = canvas_shape
        self.canvas_x_range = canvas_x_range
        self.canvas_y_range = canvas_y_range
        self.canvas_bg_color = canvas_bg_color

        self.clear_canvas()

    def get_canvas(self):
        return self.canvas

    def clear_canvas(self):
        self.canvas = np.zeros((*self.canvas_shape, 3), dtype=np.uint8)
        self.canvas[..., :] = self.canvas_bg_color

    def get_canvas_coords(self, xy):
        """
        Args:
            xy (ndarray): (N, 2+) array of coordinates. Additional columns
                beyond the first two are ignored.

        Returns:
            canvas_xy (ndarray): (N, 2) array of xy scaled into canvas
                coordinates. Invalid locations of canvas_xy are clipped into
                range. "x" is dim0, "y" is dim1 of canvas.
            valid_mask (ndarray): (N,) boolean mask indicating which of
                canvas_xy fits into canvas.
        """
        xy = np.copy(xy)  # prevent in-place modifications
        
        # np.set_printoptions(precision=3, suppress=True)
        # print(xy.shape)
        # print(xy[0:100, 0])
        # print(xy[0:100, 1])
        # print(xy[0:100, 2])
        # # print(xy[40:44])

        x = xy[:, 0]
        y = xy[:, 1]

        # Get valid mask
        valid_mask = (
            (x > self.canvas_x_range[0])
            & (x < self.canvas_x_range[1])
            & (y > self.canvas_y_range[0])
            & (y < self.canvas_y_range[1])
        )

        # print(np.max(x))
        # print(np.min(x))
        # print(np.max(y))
        # print(np.min(y))
        # print(self.canvas_x_range)
        # print(self.canvas_y_range)
        # print(self.canvas_shape)
        # zxc

        # Rescale points
        x = (x - self.canvas_x_range[0]) / (
            self.canvas_x_range[1] - self.canvas_x_range[0]
        )
        # print(x[40:44])
        x = x * self.canvas_shape[0]
        # print(x[40:44])
        x = np.clip(np.around(x), 0, self.canvas_shape[0] - 1).astype(np.int32)
        # print(x[40:44])

        y = (y - self.canvas_y_range[0]) / (
            self.canvas_y_range[1] - self.canvas_y_range[0]
        )
        y = y * self.canvas_shape[1]
        y = np.clip(np.around(y), 0, self.canvas_shape[1] - 1).astype(np.int32)

        # Return
        canvas_xy = np.stack([x, y], axis=1)

        # print(canvas_xy.shape)
        # print(canvas_xy[40:44])
        # # zxc

        return canvas_xy, valid_mask

    def draw_canvas_points(
        self, canvas_xy, radius=-1, colors=None, colors_operand=None
    ):
        """
        Draws canvas_xy onto self.canvas.

        Args:
            canvas_xy (ndarray): (N, 2) array of *valid* canvas coordinates.
                "x" is dim0, "y" is dim1 of canvas.
            radius (Int):
                -1: Each point is visualized as a single pixel.
                r: Each point is visualized as a circle with radius r.
            colors:
                None: colors all points white.
                Tuple: RGB (0 ~ 255), indicating a single color for all points.
                ndarray: (N, 3) array of RGB values for each point.
                String: Such as "Spectral", uses a matplotlib cmap, with the
                    operand (the value cmap is called on for each point) being
                    colors_operand. If colors_operand is None, uses normalized
                    distance from (0, 0) of XY point coords.
            colors_operand (ndarray | None): (N,) array of values cooresponding
                to canvas_xy, to be used only if colors is a cmap.
        """
        if len(canvas_xy) == 0:
            return

        if colors is None:
            colors = np.full((len(canvas_xy), 3), fill_value=255, dtype=np.uint8)
        elif isinstance(colors, tuple):
            assert len(colors) == 3
            colors_tmp = np.zeros((len(canvas_xy), 3), dtype=np.uint8)
            colors_tmp[..., :] = np.array(colors)
            colors = colors_tmp
        elif isinstance(colors, np.ndarray):
            assert len(colors) == len(canvas_xy)
            colors = colors.astype(np.uint8)
        elif isinstance(colors, str):
            colors = matplotlib.cm.get_cmap(colors)
            if colors_operand is None:
                # Get distances from (0, 0) (albeit potentially clipped)
                origin_center = self.get_canvas_coords(np.zeros((1, 2)))[0][0]
                colors_operand = np.sqrt(((canvas_xy - origin_center) ** 2).sum(axis=1))

            # Normalize 0 ~ 1 for cmap
            colors_operand = colors_operand - colors_operand.min()
            colors_operand = colors_operand / colors_operand.max()

            # Get cmap colors - note that cmap returns (*input_shape, 4), with
            # colors scaled 0 ~ 1
            colors = (colors(colors_operand)[:, :3] * 255).astype(np.uint8)
        else:
            raise Exception(
                "colors type {} was not an expected type".format(type(colors))
            )

        # direct draw on the canvas, x-> height, y->width
        if radius == -1:
            # self.canvas[canvas_xy[:, 0], canvas_xy[:, 1], :] = colors
            self.canvas[canvas_xy[:, 1], canvas_xy[:, 0], :] = colors
        
        # draw with cv2 requires x->horizontal (width), y-> vertical height
        # change from xy coordinate to yx coordinate
        else:
            for color, (x, y) in zip(colors.tolist(), canvas_xy.tolist()):
                # self.canvas = cv2.circle(
                #     self.canvas, (y, x), radius, color, -1, lineType=cv2.LINE_AA
                # )
                self.canvas = cv2.circle(
                    self.canvas, (x, y), radius, color, -1, lineType=cv2.LINE_AA
                )

    def draw_boxes(
        self,
        boxes=None,
        corners=None,
        colors=None,
        texts=None,
        box_line_thickness=2,
        box_text_size=0.5,
        text_corner=0,
    ):
        """
        Draws a set of boxes onto the canvas.
        Args:
            boxes (ndarray): Can either be of shape:
                (N, 7): Then, assumes (x, y, z, x_size, y_size, z_size, yaw)
                (N, 5): Then, assumes (x, y, x_size, y_size, yaw)
                Everything is in the same coordinate system as points
                (not canvas coordinates)
            colors:
                None: colors all points white.
                Tuple: RGB (0 ~ 255), indicating a single color for all points.
                ndarray: (N, 3) array of RGB values for each point.
            texts (List[String]): Length N; text to write next to boxes.
            box_line_thickness (int): cv2 line/text thickness
            box_text_size (float): cv2 putText size
            text_corner (int): 0 ~ 3. Which corner of 3D box to write text at.
        """
        num_boxes = len(boxes) if boxes is not None else len(corners)

        # Setup colors
        if colors is None:
            colors = np.full((num_boxes, 3), fill_value=255, dtype=np.uint8)
        elif isinstance(colors, tuple):
            assert len(colors) == 3
            colors_tmp = np.zeros((num_boxes, 3), dtype=np.uint8)
            colors_tmp[..., : len(colors)] = np.array(colors)
            colors = colors_tmp
        elif isinstance(colors, np.ndarray):
            assert len(colors) == num_boxes
            colors = colors.astype(np.uint8)
        else:
            raise Exception(
                "colors type {} was not an expected type".format(type(colors))
            )

        if boxes is not None:
            boxes = np.copy(boxes)  # prevent in-place modifications
            assert len(boxes.shape) == 2

            if boxes.shape[-1] == 7:
                boxes = boxes[:, [0, 1, 3, 4, 6]]
            else:
                assert boxes.shape[-1] == 5

            ## Get the BEV four corners
            # Get BEV 4 corners in box canonical coordinates
            bev_corners = (
                np.array([[[0.5, 0.5], [-0.5, 0.5], [-0.5, -0.5], [0.5, -0.5]]])
                * boxes[:, None, [2, 3]]
            )  # N x 4 x 2
            # print(bev_corners[10])

            # Get rotation matrix from yaw
            rot_sin = np.sin(boxes[:, -1])
            rot_cos = np.cos(boxes[:, -1])
            rot_matrix = np.stack(
                [[rot_cos, -rot_sin], [rot_sin, rot_cos]]
            )  # 2 x 2 x N

            # Rotate BEV 4 corners. Result: N x 4 x 2
            bev_corners = np.einsum("aij,jka->aik", bev_corners, rot_matrix)
            # print(bev_corners[10])

            # Translate BEV 4 corners
            bev_corners = bev_corners + boxes[:, None, [0, 1]]
        elif corners is not None:
            bev_corners = corners

        # print(bev_corners.shape)
        # print(bev_corners[10:13])

        ## Transform BEV 4 corners to canvas coords
        bev_corners_canvas, valid_mask = self.get_canvas_coords(
            bev_corners.reshape(-1, 2)
        )
        # print(bev_corners_canvas[40:52])

        bev_corners_canvas = bev_corners_canvas.reshape(*bev_corners.shape)
        valid_mask = valid_mask.reshape(*bev_corners.shape[:-1])
        # print(bev_corners_canvas[10:13])

        # At least 1 corner in canvas to draw.
        valid_mask = valid_mask.sum(axis=1) > 0
        bev_corners_canvas = bev_corners_canvas[valid_mask]
        if texts is not None:
            texts = np.array(texts)[valid_mask]
        if colors is not None:
            colors = np.array(colors)[valid_mask]

        ## Draw onto canvas
        # Draw the outer boundaries
        idx_draw_pairs = [(0, 1), (1, 2), (2, 3), (3, 0)]
        for i, (color, curr_box_corners) in enumerate(
            zip(colors.tolist(), bev_corners_canvas)
        ):
            curr_box_corners = curr_box_corners.astype(np.int32)
            # if i >= 10 and i < 13:
                # print(curr_box_corners[:, ::-1])

            # change xy to yx coordinate as the cv2 requires
            for start, end in idx_draw_pairs:
                self.canvas = cv2.line(
                    self.canvas,
                    tuple(curr_box_corners[start].tolist()),
                    tuple(curr_box_corners[end].tolist()),
                    # tuple(curr_box_corners[start][::-1].tolist()),
                    # tuple(curr_box_corners[end][::-1].tolist()),
                    color=color,
                    thickness=box_line_thickness,
                )
            if texts is not None:
                self.canvas = cv2.putText(
                    self.canvas,
                    str(texts[i]),
                    # tuple(curr_box_corners[text_corner][::-1].tolist()),
                    tuple(curr_box_corners[text_corner].tolist()),
                    cv2.FONT_HERSHEY_SIMPLEX,
                    box_text_size,
                    color=color,
                    thickness=box_line_thickness,
                )