import pdb import cv2 import numpy as np def get_rgbd_frame(color_path, depth_path, dtype='rgbcolormap', depth_clip=False): ''' read RGB and depth images max_depth = 10 meter, in the most frames in CDTB and DepthTrack , the depth of target is smaller than 10 m When on CDTB and DepthTrack testing, we use this depth clip ''' if color_path: rgb = cv2.imread(color_path) rgb = cv2.cvtColor(rgb, cv2.COLOR_BGR2RGB) else: rgb = None if depth_path: dp = cv2.imread(depth_path, -1) if depth_clip: max_depth = min(np.median(dp) * 3, 10000) dp[dp>max_depth] = max_depth else: dp = None if dtype == 'color': img = rgb elif dtype == 'raw_depth': img = dp elif dtype == 'colormap': dp = cv2.normalize(dp, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX) dp = np.asarray(dp, dtype=np.uint8) img = cv2.applyColorMap(dp, cv2.COLORMAP_JET) elif dtype == '3xD': dp = cv2.normalize(dp, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX) dp = np.asarray(dp, dtype=np.uint8) img = cv2.merge((dp, dp, dp)) elif dtype == 'normalized_depth': dp = cv2.normalize(dp, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX) img = np.asarray(dp, dtype=np.uint8) elif dtype == 'rgbcolormap': dp = cv2.normalize(dp, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX) dp = np.asarray(dp, dtype=np.uint8) colormap = cv2.applyColorMap(dp, cv2.COLORMAP_JET) img = cv2.merge((rgb, colormap)) elif dtype == 'rgb3d': dp = cv2.normalize(dp, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX) dp = np.asarray(dp, dtype=np.uint8) dp = cv2.merge((dp, dp, dp)) img = cv2.merge((rgb, dp)) elif dtype == 'rgbrgb': dp = cv2.cvtColor(dp, cv2.COLOR_BGR2RGB) img = cv2.merge((rgb, dp)) else: print('No such dtype !!! ') img = None return img def get_x_frame(color_path, depth_path, dtype='rgbcolormap', depth_clip=False): ''' read RGB and depth images get_rgbd_frame max_depth = 10 meter, in the most frames in CDTB and DepthTrack , the depth of target is smaller than 10 m When on CDTB and DepthTrack testing, we use this depth clip ''' if color_path: rgb = cv2.imread(color_path) rgb = cv2.cvtColor(rgb, cv2.COLOR_BGR2RGB) else: rgb = None if depth_path: dp = cv2.imread(depth_path, -1) if depth_clip: max_depth = min(np.median(dp) * 3, 10000) dp[dp > max_depth] = max_depth else: dp = None if dtype == 'color': img = rgb elif dtype == 'raw_x': img = dp elif dtype == 'colormap': dp = cv2.normalize(dp, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX) dp = np.asarray(dp, dtype=np.uint8) img = cv2.applyColorMap(dp, cv2.COLORMAP_JET) elif dtype == '3x': dp = cv2.normalize(dp, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX) dp = np.asarray(dp, dtype=np.uint8) img = cv2.merge((dp, dp, dp)) elif dtype == 'normalized_x': dp = cv2.normalize(dp, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX) img = np.asarray(dp, dtype=np.uint8) elif dtype == 'rgbcolormap': dp = cv2.normalize(dp, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX) dp = np.asarray(dp, dtype=np.uint8) colormap = cv2.applyColorMap(dp, cv2.COLORMAP_JET) # (h,w) -> (h,w,3) img = cv2.merge((rgb, colormap)) # (h,w,6) elif dtype == 'rgb3x': dp = cv2.normalize(dp, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX) dp = np.asarray(dp, dtype=np.uint8) dp = cv2.merge((dp, dp, dp)) img = cv2.merge((rgb, dp)) elif dtype == 'rgbrgb': dp = cv2.cvtColor(dp, cv2.COLOR_BGR2RGB) img = cv2.merge((rgb, dp)) else: print('No such dtype !!! ') img = None return img class p_config(object): grabcut_extra = 50 grabcut_rz_threshold = 300 grabcut_rz_factor = 1.5 minimun_target_pixels = 16 grabcut_iter = 3 radius = 500 def get_layered_image_by_depth(depth_image, target_depth, dtype='centered_colormap'): p = p_config() if target_depth is not None: low = max(target_depth - p.radius, 0) high = target_depth + p.radius layer = depth_image.copy() layer[layer < low] = high + 10 layer[layer > high] = high + 10 else: layer = depth_image.copy() layer = remove_bubbles(layer, bubbles_size=200) if dtype == 'centered_colormap': layer = cv2.normalize(layer, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_32F) layer = np.asarray(layer, dtype=np.uint8) layer = cv2.applyColorMap(layer, cv2.COLORMAP_JET) elif dtype == 'centered_normalized_depth': layer = cv2.normalize(layer, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_32F) layer = np.asarray(layer, dtype=np.uint8) layer = cv2.merge((layer, layer, layer)) elif dtype == 'centered_raw_depth': layer = np.asarray(layer) layer = np.stack((layer, layer, layer), axis=2) return layer def remove_bubbles(image, bubbles_size=100): try: binary_map = (image > 0).astype(np.uint8) nb_components, output, stats, centroids = cv2.connectedComponentsWithStats(binary_map, connectivity=8) sizes = stats[1:, -1] nb_components = nb_components - 1 mask = np.zeros((image.shape), dtype=np.uint8) for i in range(0, nb_components): if sizes[i] >= bubbles_size: mask[output == i + 1] = 1 if len(image.shape) > 2: image = image * mask[:, :, np.newaxis] else: image = image * mask except: pass return image def get_target_depth(depth, target_box): ''' To estimate the target depth by using cv2.grabCut ''' p = p_config() H, W = depth.shape target_box = [int(bb) for bb in target_box] x0, y0, w0, h0 = target_box x0 = max(x0, 0) y0 = max(y0, 0) x1 = min(x0 + w0, W) y1 = min(y0 + h0, H) possible_target = depth[y0:y1, x0:x1] median_depth = np.median(possible_target) + 10 bubbles_size = int(target_box[2] * target_box[3] * 0.1) try: ''' add the surrounding extra pixels as the background ''' extra_y0 = max(y0 - p.grabcut_extra, 0) extra_x0 = max(x0 - p.grabcut_extra, 0) extra_y1 = min(y1 + p.grabcut_extra, H) extra_x1 = min(x1 + p.grabcut_extra, W) rect_x0 = x0 - extra_x0 rect_y0 = y0 - extra_y0 rect_x1 = min(rect_x0 + w0, extra_x1) rect_y1 = min(rect_y0 + h0, extra_y1) rect = [rect_x0, rect_y0, rect_x1 - rect_x0, rect_y1 - rect_y0] target_patch = depth[extra_y0:extra_y1, extra_x0:extra_x1] target_patch = np.nan_to_num(target_patch, nan=np.max(target_patch)) ''' filter depth image, then convert to colormap ''' image = target_patch.copy() image[image > median_depth * 2] = median_depth * 2 # !!!!!!!!!! image[image < 10] = median_depth * 2 '''To downsample the target_patch in order to speed up the cv2.grabCut''' i_H, i_W = image.shape rz_factor = p.grabcut_rz_factor if min(i_W, i_H) > p.grabcut_rz_threshold else 1 rect_rz = [int(rt // rz_factor) for rt in rect] rz_dim = (int(i_W // rz_factor), int(i_H // rz_factor)) image = cv2.resize(image, rz_dim, interpolation=cv2.INTER_AREA) image = remove_bubbles(image, bubbles_size=bubbles_size) image = cv2.normalize(image, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_32F) image = np.asarray(image, dtype=np.uint8) image = cv2.applyColorMap(image, cv2.COLORMAP_JET) ''' 0-pixels and 2-pixels are background(set to 0), 1-pixels and 3-pixels are foreground(set to 1)''' mask = np.zeros(image.shape[:2], np.uint8) bgdModel = np.zeros((1, 65), np.float64) fgdModel = np.zeros((1, 65), np.float64) cv2.grabCut(image, mask, rect_rz, bgdModel, fgdModel, p.grabcut_iter, cv2.GC_INIT_WITH_RECT) mask2 = np.where((mask == 2) | (mask == 0), 0, 1).astype('uint8') mask2 = remove_bubbles(mask2, bubbles_size=bubbles_size) mask2 = cv2.resize(mask2, (i_W, i_H), interpolation=cv2.INTER_AREA) ''' to get the target depth values ''' target_pixels = target_patch * mask2 target_pixels = target_pixels.flatten() target_pixels.sort() target_pixels = target_pixels[target_pixels > 0] if len(target_pixels) > p.minimun_target_pixels: hist, bin_edges = np.histogram(target_pixels, bins=20) peak_idx = np.argmax(hist) selected_target_pixels = target_pixels target_depth_low = bin_edges[peak_idx] target_depth_high = bin_edges[peak_idx + 1] selected_target_pixels = selected_target_pixels[selected_target_pixels <= target_depth_high] selected_target_pixels = selected_target_pixels[selected_target_pixels >= target_depth_low] target_depth = np.median(selected_target_pixels) else: target_depth = median_depth except: target_depth = median_depth return target_depth