mmdet3d_plugin/datasets/utils/transform.py

import copy
from typing import Dict, Optional

import numpy as np

import utils.calibration as calib_utils

################## box in the object's own coordinate


def create_bb_points(size: np.ndarray) -> np.ndarray:
    """Create bounding box's 8 corners in the object coordinate

    Note: we assume that the location of the object is at the center
    of the object, not the bottom center

    Args:
        size: (3, ), length, width and height, obtained from
        carla_utils.vec2np(obj.bounding_box.extent)

    Returns:
        corners: 8 x 4, in the object coordinate

    Carla use left-handed coordinate, world coordinate:
        +x -> front
        +y -> right
        +z -> up
        4 -------- 5
       /|         /|
      7 -------- 6 .
      | |        | |
      . 0 -------- 1
      |/         |/
      3 -------- 2
    e.g., if take points of 0, 1, 2, 3, it will return the box in bottom
    if take points of 0, 3, 4, 7, it will return the box on the left
    """

    # decompose object size into length, width, height
    l, w, h = size

    # create corner points
    corners = np.zeros((8, 4))
    corners[0, :] = np.array([l / 2, -w / 2, -h / 2, 1])
    corners[1, :] = np.array([l / 2, w / 2, -h / 2, 1])
    corners[2, :] = np.array([-l / 2, w / 2, -h / 2, 1])
    corners[3, :] = np.array([-l / 2, -w / 2, -h / 2, 1])
    corners[4, :] = np.array([l / 2, -w / 2, h / 2, 1])
    corners[5, :] = np.array([l / 2, w / 2, h / 2, 1])
    corners[6, :] = np.array([-l / 2, w / 2, h / 2, 1])
    corners[7, :] = np.array([-l / 2, -w / 2, h / 2, 1])

    return corners


def box_in_obj_to_world(box_in_object: np.ndarray, transform: np.ndarray) -> np.ndarray:
    """Convert object boxes in its own coordinate to world coordinate

    Args:
        corners: 8 x 4 in the object coordinate
        transform: 4 x 4, object to world transformation
    """

    box_in_world = np.dot(transform, np.transpose(box_in_object)).transpose()  #  8 x 4
    return box_in_world


################## box in the world coordinate


def get_box_in_world(
    location: np.ndarray, rotation: np.ndarray, size: np.ndarray
) -> np.ndarray:
    """Return 3D bounding box in world coordinate"""

    # get box in the object's own coordinate
    box_in_object: np.ndarray = create_bb_points(size)  # 8 x 4

    # apply transformation to the world coordinate
    transform: np.ndarray = calib_utils.create_transform(location, rotation)
    box_in_world: np.ndarray = box_in_obj_to_world(box_in_object, transform)  # 8 x 4

    return box_in_world


################## box in the ego vehicle's coordinate


def box_in_world_to_ego(
    box_in_world: np.ndarray, ego_transform: np.ndarray
) -> np.ndarray:
    """Convert bbox in the world coordinate to the ego coordinate

    Args:
        box_in_world: (8, 4), 8 corners in the world coordinate
        ego_transform: (4, 4), transformation from ego to the world coordinate

    Returns:
        box_in_ego: (8, 4), 8 corners in the ego coordinate
    """

    # avoid in-place operation
    box_in_world = copy.copy(box_in_world)  # 8 x 4

    # apply transformation
    world2ego_transform = np.linalg.inv(ego_transform)  # 4 x 4
    box_in_ego = np.dot(
        world2ego_transform, np.transpose(box_in_world)
    ).transpose()  # 8 x 4

    return box_in_ego


################## box in the LiDAR's coordinate


def box_in_ego_to_lidar(box_in_ego: np.ndarray, ego2lidar: Optional[np.ndarray] = None) -> np.ndarray:
    """Convert bbox in the ego coordinate to the lidar coordinate
    
    Arguments
        box_in_ego: 8 x 4 
        ego2lidar: 4 x 4
    """

    # avoid in-place operation
    box_in_ego = copy.copy(box_in_ego)  # 8 x 4

    # get transformation by default
    if ego2lidar is None:
        ego2lidar = np.linalg.inv(calib_utils.lidar_to_ego_transform())  # 4 x 4

    # apply transformation
    box_in_lidar = np.dot(ego2lidar, np.transpose(box_in_ego)).transpose()  # 8 x 4

    return box_in_lidar


def box_in_lidar_to_ego(box_in_lidar: np.ndarray, lidar2ego: Optional[np.ndarray] = None) -> np.ndarray:
    """Convert bbox in the lidar coordinate to the ego coordinate
    
    Arguments
        box_in_lidar: 8 x 4 
        lidar2ego: 4 x 4
    """

    # avoid in-place operation
    box_in_lidar = copy.copy(box_in_lidar)  # 8 x 4

    # get transformation by default
    if lidar2ego is None:
        lidar2ego = calib_utils.lidar_to_ego_transform()
        # print(lidar2ego)
        assert False, 'No lidar2ego is provided!!!'
        # asd
    # qwe
    # print('using provided')

    # apply transformation
    box_in_ego = np.dot(lidar2ego, np.transpose(box_in_lidar)).transpose()  # 8 x 4

    return box_in_ego


################## box in the camera/image's coordinate


def box_in_ego_to_camera(box_in_ego: np.ndarray, cam_param: Dict) -> np.ndarray:
    """Convert bbox in the ego coordinate to the camera coordinate"""

    # avoid in-place operation
    box_in_ego = copy.copy(box_in_ego)  # 8 x 4

    # get transformation by default
    if 'ego2cam' in cam_param:
        ego2cam: np.narray = cam_param['ego2cam']
    else:
        ego2cam = np.linalg.inv(
            calib_utils.camera_to_ego_transform(cam_param)
        )  # 4 x 4

    # apply transformation
    box_in_camera = np.dot(ego2cam, np.transpose(box_in_ego)).transpose()  # 8 x 4

    return box_in_camera


def box_projection_to_image(
    box_in_camera: np.ndarray, 
    intrinsics: np.ndarray, 
    front_dist_threshold: float = 1.5,
    left_hand_system: bool = True,
) -> np.ndarray:
    """Project 3D box in camera coordinate to 2D box in image coordinate

    Args:
        box_in_camera: (8, 4), 3D box in the camera coordinate
        intrinsics: (4, 4) camera intrinsic matrix
        front_dist_threshold: used to filter out bad project due to too close

    ReturnsL
        box_2d_in_image: (8, 2), 2D box in the image coordinate
    """

    box_in_camera = copy.copy(box_in_camera).transpose()  # 4 x 8

    # camera rotation correction, from left to right hand coordinate system
    # change to camera coordinate as in other datasets such as KITTI
    if left_hand_system:
        box_in_camera = np.vstack(
            (
                box_in_camera[1, :],
                -box_in_camera[2, :],
                box_in_camera[0, :],
                box_in_camera[3, :],
            )
        )  # 4 x 8

    # apply projection and normalization of the homogeneous coordinate
    box_2d_in_image = (intrinsics @ box_in_camera).T  # 8 x 4
    box_2d_in_image[:, 0] = box_2d_in_image[:, 0] / box_2d_in_image[:, 2]
    box_2d_in_image[:, 1] = box_2d_in_image[:, 1] / box_2d_in_image[:, 2]

    # check if the box is in front of camera
    if np.any(box_2d_in_image[:, 2] > front_dist_threshold):
        box_2d_in_image = np.array(box_2d_in_image)[:, :2]
        return box_2d_in_image  # 8 x 2
    else:
        return None