from typing import Dict, List, Optional, Tuple, Union import cv2 import utils.transform as box_transform import utils.calibration as calib_utils import utils.bev as bev_vis import utils.canvas_3d as vis_3d import utils.color as color_vis import matplotlib.pyplot as plt plt.switch_backend('agg') import numpy as np import scipy from PIL import Image from pathlib import Path import os def get_color_object(obj_id: int, color_list: List) -> np.array: """Get color for each object for vis, w.r.t. the ID""" color_float: Tuple[float, float, float] = color_list[obj_id % len(color_list)] color = np.array( [ int(color_float[0] * 255), int(color_float[1] * 255), int(color_float[2] * 255), ] ) return color def vis_box_on_lidar( lidar: Union[str, np.ndarray], save_path: str, bbox_det_in_ego_list: Optional[List[np.ndarray]] = None, bbox_gt_in_ego_list: Optional[List[np.ndarray]] = None, id_list: Optional[List[int]] = None, state_list: Optional[List[np.ndarray]] = None, fut_traj: Optional[List[np.ndarray]] = None, # xy coordinate fut_traj_mask: Optional[List[np.ndarray]] = None, pre_traj: Optional[List[np.ndarray]] = None, pre_traj_mask: Optional[List[np.ndarray]] = None, ego2lidar: Optional[np.ndarray] = None, bev: bool = True, box_order_nuscenes: bool = False, left_hand: bool = True, # carla vis_text: bool = True, command: Optional[int] = 2, command_dict: Optional[Dict[int, str]] = {0: 'TURN RIGHT', 1: 'TURN LEFT', 2: 'KEEP FORWARD'}, bev_map: Optional[np.ndarray] = None, # H x W x 3 ) -> None: """Visualize detection/GT boxes on the LiDAR point cloud Args: lidar: str -> path to the data, np array -> N x 3 data already loaded state_list: a list of array including velocity/acceleration/angular_velocity fut_traj: N x 12 x 4 or N, 12 x 4 past_traj: N x 8 x 2 or N, 8 x 2 Note that carla lidar coordinate is +x -> left +y -> front +z -> up """ # load lidar if isinstance(lidar, str): # try: # lidar = np.load(lidar, allow_pickle=True)[..., :3] # XYZI, N x 3 now # except: lidar = np.fromfile(lidar, dtype=np.float32) lidar = lidar.reshape(-1, 4) # N x 4 lidar = lidar[:, :3] # N x 3 now else: assert isinstance(lidar, np.ndarray), "the detection is either str or dict" # pre-load color for each object when visualizing tracklets # only when id_list is provided and visualizing either GT/tracklets if id_list is not None and ( bbox_det_in_ego_list is None or bbox_gt_in_ego_list is None ): # get the list of colors to choose for each tracklet/prediction color_list: List[Tuple[float, float, float]] = color_vis.random_colors() if bbox_gt_in_ego_list is None: num_obj = len(bbox_det_in_ego_list) else: num_obj = len(bbox_gt_in_ego_list) colors = np.zeros((num_obj, 3), dtype=np.uint8) # N x 3 assert len(id_list) == num_obj, "number of objects is wrong" for obj_index, obj_id in enumerate(id_list): color: np.ndarray = get_color_object(obj_id, color_list) # (3, ) colors[obj_index, :] = color else: colors = None # convert the ID_list to list of strings to visualize # only if id_list is provided text_corner = 1 if id_list is not None and vis_text: texts = [str(obj_id) for obj_id in id_list] # add the list of states to the text if state_list is not None: for index in range(len(state_list)): states: Union[np.ndarray, List] = state_list[index] state_str: str = " ".join(["%.2f" % state for state in states]) texts[index] = texts[index] + " " + state_str else: texts = None # visualize lidar points only, default color ORANGE if bev: canvas = bev_vis.Canvas_BEV() canvas_xy, valid_mask = canvas.get_canvas_coords(lidar) # Get Canvas coords canvas.draw_canvas_points(canvas_xy[valid_mask]) # Only draw valid points else: # 3D canvas = vis_3d.Canvas_3D() canvas_xyz, valid_mask = canvas.get_canvas_coords(lidar) # Get Canvas coords color_based_on_intensity = ( (lidar[valid_mask, -1] * 255).astype("uint8").reshape((-1, 1)) ) # N x 1 color_based_on_intensity = np.repeat( color_based_on_intensity, 3, axis=1 ) # N x 3 canvas.draw_canvas_points( canvas_xyz[valid_mask], colors=color_based_on_intensity ) # Only draw valid points # extract height from the bbox if (bbox_det_in_ego_list is not None) or (bbox_gt_in_ego_list is not None): if bbox_det_in_ego_list is not None: num_obj = len(bbox_det_in_ego_list) else: num_obj = len(bbox_gt_in_ego_list) height_all_lidar = np.zeros((num_obj), dtype='float32') # visualize detection bbox on lidar, default color WHITE if bbox_det_in_ego_list is not None: bbox_det_in_lidar_list: List[np.ndarray] = list() count = 0 box_in_ego: np.ndarray for box_in_ego in bbox_det_in_ego_list: # convert box in ego to lidar coordinate box_in_lidar: np.ndarray = box_transform.box_in_ego_to_lidar( box_in_ego, ego2lidar=ego2lidar, ) # 8 x 4 height_all_lidar[count] = np.average(box_in_lidar, axis=0)[2] if bev: if box_order_nuscenes: box_in_lidar = box_in_lidar[ [1, 2, 6, 5], :2 ] # 4 x 2, nuScenes boxinstances else: box_in_lidar = box_in_lidar[:4, :2] # 4 x 2, take only BEV corners else: box_in_lidar = box_in_lidar[:, :3] # 8 x 3, take only xyz bbox_det_in_lidar_list.append(np.array(box_in_lidar)) count += 1 if len(bbox_det_in_lidar_list) > 0: bbox_det_in_lidar = np.stack(bbox_det_in_lidar_list, axis=0) # N x 4 x 2 if colors is None: colors_det = np.full( (len(bbox_det_in_lidar), 3), fill_value=0, dtype=np.uint8 ) colors_det[:, :2] = 255 # (N x 3) else: colors_det = colors canvas.draw_boxes( corners=bbox_det_in_lidar, colors=colors_det, texts=texts, text_corner=text_corner, ) # Draw Boxes # visualize GT bbox on lidar, default color BLUE if bbox_gt_in_ego_list is not None: bbox_gt_in_lidar_list: List[np.ndarray] = list() box_in_ego: np.ndarray count = 0 for box_in_ego in bbox_gt_in_ego_list: # convert box in ego to lidar coordinate box_in_lidar: np.ndarray = box_transform.box_in_ego_to_lidar( box_in_ego, ego2lidar=ego2lidar, ) # 8 x 4 height_all_lidar[count] = np.average(box_in_lidar, axis=0)[2] # convert to BEV if bev: if box_order_nuscenes: box_in_lidar = box_in_lidar[ [1, 2, 6, 5], :2 ] # 4 x 2, nuScenes boxinstances else: box_in_lidar = box_in_lidar[:4, :2] # 4 x 2, take only BEV corners # box_in_lidar = box_in_lidar[[0, 3, 4, 7], :2] # 4 x 2, take left side corners # box_in_lidar = box_in_lidar[[0, 1, 4, 5], :2] # 4 x 2, take front side corners else: box_in_lidar = box_in_lidar[:, :3] # if count >= 10 and count < 13: # print("box_in_lidar", box_in_lidar) bbox_gt_in_lidar_list.append(np.array(box_in_lidar)) count += 1 if len(bbox_gt_in_lidar_list) > 0: bbox_gt_in_lidar = np.stack(bbox_gt_in_lidar_list, axis=0) # N x 4 x 2 if colors is None: colors_gt = np.full( (len(bbox_gt_in_lidar), 3), fill_value=0, dtype=np.uint8 ) colors_gt[:, 2] = 255 # N x 3 else: colors_gt = colors canvas.draw_boxes( corners=bbox_gt_in_lidar, colors=colors_gt, texts=texts, text_corner=text_corner, ) # Draw Boxes # draw points for past/future trajectories in lidar coordinate if fut_traj is not None: fut_traj_obj: np.ndarray # F x 4 obj_index = 0 for fut_traj_obj in fut_traj: # add height to the trajectory in the 3D vis cases if fut_traj_obj.shape[1] == 2 and not bev: fut_traj_obj_addh = np.concatenate([fut_traj_obj, np.zeros((fut_traj_obj.shape[0], 1))], axis=1) fut_traj_obj_addh[:, -1] = height_all_lidar[obj_index] fut_traj_obj = fut_traj_obj_addh # Get Canvas coords and mask within the visualization range canvas_xy, valid_mask = canvas.get_canvas_coords( fut_traj_obj ) # canvas_xy: N x 2, valid_mask: N valid_mask = valid_mask.astype("bool") # get valid mask for every timestamp fut_traj_mask_obj: np.ndarray = fut_traj_mask[obj_index, :, 0].astype( "bool" ) # N mask_combined = fut_traj_mask_obj & valid_mask # visualize canvas.draw_canvas_points( canvas_xy[mask_combined], colors=tuple(colors[obj_index]), radius=2 ) # Only draw valid points obj_index += 1 if pre_traj is not None: pre_traj_obj: np.ndarray # F x 4 obj_index = 0 for pre_traj_obj in pre_traj: # add height to the trajectory in the 3D vis cases if pre_traj_obj.shape[1] == 2 and not bev: pre_traj_obj_addh = np.concatenate([pre_traj_obj, np.zeros((pre_traj_obj.shape[0], 1))], axis=1) pre_traj_obj_addh[:, -1] = height_all_lidar[obj_index] pre_traj_obj = pre_traj_obj_addh canvas_xy, valid_mask = canvas.get_canvas_coords( pre_traj_obj ) # Get Canvas coords valid_mask = valid_mask.astype("bool") # get valid mask for every timestamp pre_traj_mask_obj: np.ndarray = pre_traj_mask[obj_index, :, 0].astype( "bool" ) # N mask_combined = pre_traj_mask_obj & valid_mask canvas.draw_canvas_points( canvas_xy[mask_combined], colors=tuple(colors[obj_index]), radius=2 ) # Only draw valid points obj_index += 1 # rotate to make the view consistent with the image # now +x -> left (ego), +y -> front (ego) if bev: canvas.canvas = np.flip(canvas.canvas, axis=(0)) # now +x -> right (ego), +y -> front (ego) if left_hand: canvas.canvas = np.flip(canvas.canvas, axis=(1)) # now +x -> right (ego), +y -> back (ego) # draw figure plt.figure(figsize=(20, 20)) plt.axis("off") plt.imshow(canvas.canvas) plt.tight_layout() # add text for commands plt.text(30, 50, command_dict[int(command)], fontsize=45, color=(1, 1, 1)) # save figure save_dir = Path(save_path).resolve().parent if not os.path.exists(save_dir): os.makedirs(save_dir) plt.savefig(save_path, transparent=False) plt.close() # overlay visualization with map if bev_map is not None and bev: traj_vis = cv2.imread(save_path) # convert the bev map to match with the higher-resolution visualization image bev_map = cv2.resize(bev_map, traj_vis.shape[1::-1]) # 2000 x 2000 bev_map = (bev_map * 255).astype('uint8') # make up the third dimension if only one channel is given if len(bev_map.shape) == 2: bev_map = np.expand_dims(bev_map, axis=-1) # 2000 x 2000 x 1 bev_map = np.repeat(bev_map, 3, axis=-1) # 2000 x 2000 x 3 # synchronize with the lidar visualization if bev: bev_map = np.flip(bev_map, axis=0) # H x W x 3 if left_hand: bev_map = np.flip(bev_map, axis=1) # H x W x 3 # blend & save dst = cv2.addWeighted(traj_vis, 0.7, bev_map, 0.3, 0) cv2.imwrite(save_path, dst) def vis_box_on_camera( cameras: Dict[str, np.ndarray], save_path: str, cam_param_all: Dict[str, Dict], bbox_det_in_ego_list: Optional[List[np.ndarray]] = None, bbox_gt_in_ego_list: Optional[List[np.ndarray]] = None, left_hand_system: bool = True, ) -> None: """Visualize detection/GT boxes on the multiple cameras Args: camera: keys can be ["left", "front", "right", "back"] values are H x W x 3 """ # process each camera cam_name: str cam_param: Dict[str, float] for cam_name, cam_param in cam_param_all.items(): # if we already have ego2cam for visualization, no need to construct the transform if "intrinsics" not in cam_param: intrinsics: np.ndarray = calib_utils.get_camera_intrinsics(cam_param, dim=4) else: intrinsics: np.ndarray = cam_param["intrinsics"] # visualize each box for detection, default color ORANGE if bbox_det_in_ego_list is not None: box_in_ego: np.ndarray for box_in_ego in bbox_det_in_ego_list: # first convert to the camera coordinate # and then 3D -> 2D projection to the image coordinate box_in_camera: np.ndarray = box_transform.box_in_ego_to_camera( box_in_ego, cam_param ) # 8 x 4 # camera projection box_2d_in_image: np.ndarray = box_transform.box_projection_to_image( box_in_camera, intrinsics, left_hand_system=left_hand_system, ) # 8 x 2 # check if box is in front of the camera for visualization if box_2d_in_image is not None: cameras[cam_name], _ = vis_box_on_single_image( cameras[cam_name], box_2d_in_image, img_size=(cam_param["height"], cam_param["width"]), color=(255, 255, 0), thickness=2, ) # visualize each box for GT, default color BLUE if bbox_gt_in_ego_list is not None: box_in_ego: np.ndarray for box_in_ego in bbox_gt_in_ego_list: # first convert to the camera coordinate # and then 3D -> 2D projection to the image coordinate box_in_camera: np.ndarray = box_transform.box_in_ego_to_camera( box_in_ego, cam_param ) # 8 x 4 # camera projection box_2d_in_image: np.ndarray = box_transform.box_projection_to_image( box_in_camera, intrinsics, left_hand_system=left_hand_system, ) # 8 x 2 # check if box is in front of the camera for visualization if box_2d_in_image is not None: cameras[cam_name], _ = vis_box_on_single_image( cameras[cam_name], box_2d_in_image, img_size=(cam_param["height"], cam_param["width"]), color=(0, 0, 255), thickness=2, ) # visualize all images together # TODO (Xinshuo): remove this assumption # assume all cameras have the same width and height image = np.concatenate(list(cameras.values()), axis=1) # H x 3W x 3 image = image[:, :, ::-1] # convert bgr to rgb image = Image.fromarray(image) image = image.resize( (cam_param["width"] * len(cameras.keys()), cam_param["height"]) ) save_dir = Path(save_path).resolve().parent if not os.path.exists(save_dir): os.makedirs(save_dir) image.save(save_path) def check_outside_image(x, y, height, width): if x < 0 or x >= width: return True if y < 0 or y >= height: return True def vis_box_on_single_image( image, qs, img_size=(900, 1600), color=(255, 255, 255), thickness=4 ): """Draw 3d bounding box in image qs: (8,2) array of vertices for the 3d box in following order: 1 -------- 0 /| /| 2 -------- 3 . | | | | . 5 -------- 4 |/ |/ 6 -------- 7 """ # if 6 points of the box are outside the image, then do not draw pts_outside = 0 for index in range(8): check = check_outside_image( qs[index, 0], qs[index, 1], img_size[0], img_size[1] ) if check: pts_outside += 1 if pts_outside >= 6: return image, False # actually draw if qs is not None: qs = qs.astype(np.int32) for k in range(0, 4): i, j = k, (k + 1) % 4 cv2.line( image, (qs[i, 0], qs[i, 1]), (qs[j, 0], qs[j, 1]), color, thickness, cv2.LINE_AA, ) # use LINE_AA for opencv3 i, j = k + 4, (k + 1) % 4 + 4 cv2.line( image, (qs[i, 0], qs[i, 1]), (qs[j, 0], qs[j, 1]), color, thickness, cv2.LINE_AA, ) i, j = k, k + 4 cv2.line( image, (qs[i, 0], qs[i, 1]), (qs[j, 0], qs[j, 1]), color, thickness, cv2.LINE_AA, ) return image, True