import os import sys import re import numpy as np import cv2 from PIL import Image from imageio import imread, imwrite import torch import torch.nn.functional as F def EPE(input_flow, target_flow, mask): diff = input_flow - target_flow epe = (mask * torch.norm(diff, p=2, dim=1, keepdim=True)).sum() / (mask.sum() + 1e-6) return epe def centralize(img1, img2): rgb_mean = torch.cat([img1, img2], 2).mean(1, keepdim=True).mean(2, keepdim=True).mean(3, keepdim=True) return img1 - rgb_mean, img2 - rgb_mean, rgb_mean def warp(x, flo): flo = flo.clone() B, C, H, W = x.size() xx = torch.arange(0, W).view(1, -1).repeat(H, 1) yy = torch.arange(0, H).view(-1, 1).repeat(1, W) xx = xx.view(1, 1, H, W).repeat(B, 1, 1, 1) yy = yy.view(1, 1, H, W).repeat(B, 1, 1, 1) grid = torch.cat([xx, yy], 1).to(x) vgrid = grid + flo vgrid[:, 0, :, :] = 2.0 * vgrid[:, 0, :, :] / max(W-1, 1) - 1.0 vgrid[:, 1, :, :] = 2.0 * vgrid[:, 1, :, :] / max(H-1, 1) - 1.0 vgrid = vgrid.permute(0, 2, 3, 1) output = F.grid_sample(x, vgrid, mode='bilinear', align_corners=True) return output def resize_img(img, size): img = F.interpolate(img, size, mode='bilinear', align_corners=False) return img def resize_flow(flow, size): scale_x = size[1] / flow.shape[3] scale_y = size[0] / flow.shape[2] flow = F.interpolate(flow, size, mode='bilinear', align_corners=False) flow[:,0,:,:] *= scale_x flow[:,1,:,:] *= scale_y return flow def make_colorwheel(): """ Generates a color wheel for optical flow visualization as presented in: Baker et al. "A Database and Evaluation Methodology for Optical Flow" (ICCV, 2007) URL: http://vision.middlebury.edu/flow/flowEval-iccv07.pdf Code follows the original C++ source code of Daniel Scharstein. Code follows the the Matlab source code of Deqing Sun. Returns: np.ndarray: Color wheel """ RY = 15 YG = 6 GC = 4 CB = 11 BM = 13 MR = 6 ncols = RY + YG + GC + CB + BM + MR colorwheel = np.zeros((ncols, 3)) col = 0 # RY colorwheel[0:RY, 0] = 255 colorwheel[0:RY, 1] = np.floor(255*np.arange(0,RY)/RY) col = col+RY # YG colorwheel[col:col+YG, 0] = 255 - np.floor(255*np.arange(0,YG)/YG) colorwheel[col:col+YG, 1] = 255 col = col+YG # GC colorwheel[col:col+GC, 1] = 255 colorwheel[col:col+GC, 2] = np.floor(255*np.arange(0,GC)/GC) col = col+GC # CB colorwheel[col:col+CB, 1] = 255 - np.floor(255*np.arange(CB)/CB) colorwheel[col:col+CB, 2] = 255 col = col+CB # BM colorwheel[col:col+BM, 2] = 255 colorwheel[col:col+BM, 0] = np.floor(255*np.arange(0,BM)/BM) col = col+BM # MR colorwheel[col:col+MR, 2] = 255 - np.floor(255*np.arange(MR)/MR) colorwheel[col:col+MR, 0] = 255 return colorwheel def flow_uv_to_colors(u, v, convert_to_bgr=False): """ Applies the flow color wheel to (possibly clipped) flow components u and v. According to the C++ source code of Daniel Scharstein According to the Matlab source code of Deqing Sun Args: u (np.ndarray): Input horizontal flow of shape [H,W] v (np.ndarray): Input vertical flow of shape [H,W] convert_to_bgr (bool, optional): Convert output image to BGR. Defaults to False. Returns: np.ndarray: Flow visualization image of shape [H,W,3] """ flow_image = np.zeros((u.shape[0], u.shape[1], 3), np.uint8) colorwheel = make_colorwheel() # shape [55x3] ncols = colorwheel.shape[0] rad = np.sqrt(np.square(u) + np.square(v)) a = np.arctan2(-v, -u)/np.pi fk = (a+1) / 2*(ncols-1) k0 = np.floor(fk).astype(np.int32) k1 = k0 + 1 k1[k1 == ncols] = 0 f = fk - k0 for i in range(colorwheel.shape[1]): tmp = colorwheel[:,i] col0 = tmp[k0] / 255.0 col1 = tmp[k1] / 255.0 col = (1-f)*col0 + f*col1 idx = (rad <= 1) col[idx] = 1 - rad[idx] * (1-col[idx]) col[~idx] = col[~idx] * 0.75 # out of range # Note the 2-i => BGR instead of RGB ch_idx = 2-i if convert_to_bgr else i flow_image[:,:,ch_idx] = np.floor(255 * col) return flow_image def flow_to_color(flow_uv, clip_flow=None, convert_to_bgr=False): """ Expects a two dimensional flow image of shape. Args: flow_uv (np.ndarray): Flow UV image of shape [H,W,2] clip_flow (float, optional): Clip maximum of flow values. Defaults to None. convert_to_bgr (bool, optional): Convert output image to BGR. Defaults to False. Returns: np.ndarray: Flow visualization image of shape [H,W,3] """ assert flow_uv.ndim == 3, 'input flow must have three dimensions' assert flow_uv.shape[2] == 2, 'input flow must have shape [H,W,2]' if clip_flow is not None: flow_uv = np.clip(flow_uv, 0, clip_flow) u = flow_uv[:,:,0] v = flow_uv[:,:,1] rad = np.sqrt(np.square(u) + np.square(v)) rad_max = np.max(rad) epsilon = 1e-5 u = u / (rad_max + epsilon) v = v / (rad_max + epsilon) return flow_uv_to_colors(u, v, convert_to_bgr) def read(file): if file.endswith('.float3'): return readFloat(file) elif file.endswith('.flo'): return readFlow(file) elif file.endswith('.ppm'): return readImage(file) elif file.endswith('.pgm'): return readImage(file) elif file.endswith('.png'): return readImage(file) elif file.endswith('.jpg'): return readImage(file) elif file.endswith('.pfm'): return readPFM(file)[0] else: raise Exception('don\'t know how to read %s' % file) def write(file, data): if file.endswith('.float3'): return writeFloat(file, data) elif file.endswith('.flo'): return writeFlow(file, data) elif file.endswith('.ppm'): return writeImage(file, data) elif file.endswith('.pgm'): return writeImage(file, data) elif file.endswith('.png'): return writeImage(file, data) elif file.endswith('.jpg'): return writeImage(file, data) elif file.endswith('.pfm'): return writePFM(file, data) else: raise Exception('don\'t know how to write %s' % file) def readPFM(file): file = open(file, 'rb') color = None width = None height = None scale = None endian = None header = file.readline().rstrip() if header.decode("ascii") == 'PF': color = True elif header.decode("ascii") == 'Pf': color = False else: raise Exception('Not a PFM file.') dim_match = re.match(r'^(\d+)\s(\d+)\s$', file.readline().decode("ascii")) if dim_match: width, height = list(map(int, dim_match.groups())) else: raise Exception('Malformed PFM header.') scale = float(file.readline().decode("ascii").rstrip()) if scale < 0: # little-endian endian = '<' scale = -scale else: endian = '>' # big-endian data = np.fromfile(file, endian + 'f') shape = (height, width, 3) if color else (height, width) data = np.reshape(data, shape) data = np.flipud(data) return data, scale def writePFM(file, image, scale=1): file = open(file, 'wb') color = None if image.dtype.name != 'float32': raise Exception('Image dtype must be float32.') image = np.flipud(image) if len(image.shape) == 3 and image.shape[2] == 3: # color image color = True elif len(image.shape) == 2 or len(image.shape) == 3 and image.shape[2] == 1: # greyscale color = False else: raise Exception('Image must have H x W x 3, H x W x 1 or H x W dimensions.') file.write('PF\n' if color else 'Pf\n'.encode()) file.write('%d %d\n'.encode() % (image.shape[1], image.shape[0])) endian = image.dtype.byteorder if endian == '<' or endian == '=' and sys.byteorder == 'little': scale = -scale file.write('%f\n'.encode() % scale) image.tofile(file) def readFlow(name): if name.endswith('.pfm') or name.endswith('.PFM'): return readPFM(name)[0][:,:,0:2] f = open(name, 'rb') header = f.read(4) if header.decode("utf-8") != 'PIEH': raise Exception('Flow file header does not contain PIEH') width = np.fromfile(f, np.int32, 1).squeeze() height = np.fromfile(f, np.int32, 1).squeeze() flow = np.fromfile(f, np.float32, width * height * 2).reshape((height, width, 2)) return flow.astype(np.float32) def readImage(name): if name.endswith('.pfm') or name.endswith('.PFM'): data = readPFM(name)[0] if len(data.shape)==3: return data[:,:,0:3] else: return data return imread(name) def writeImage(name, data): if name.endswith('.pfm') or name.endswith('.PFM'): return writePFM(name, data, 1) return imwrite(name, data) def writeFlow(name, flow): f = open(name, 'wb') f.write('PIEH'.encode('utf-8')) np.array([flow.shape[1], flow.shape[0]], dtype=np.int32).tofile(f) flow = flow.astype(np.float32) flow.tofile(f) def readFloat(name): f = open(name, 'rb') if(f.readline().decode("utf-8")) != 'float\n': raise Exception('float file %s did not contain keyword' % name) dim = int(f.readline()) dims = [] count = 1 for i in range(0, dim): d = int(f.readline()) dims.append(d) count *= d dims = list(reversed(dims)) data = np.fromfile(f, np.float32, count).reshape(dims) if dim > 2: data = np.transpose(data, (2, 1, 0)) data = np.transpose(data, (1, 0, 2)) return data def writeFloat(name, data): f = open(name, 'wb') dim=len(data.shape) if dim>3: raise Exception('bad float file dimension: %d' % dim) f.write(('float\n').encode('ascii')) f.write(('%d\n' % dim).encode('ascii')) if dim == 1: f.write(('%d\n' % data.shape[0]).encode('ascii')) else: f.write(('%d\n' % data.shape[1]).encode('ascii')) f.write(('%d\n' % data.shape[0]).encode('ascii')) for i in range(2, dim): f.write(('%d\n' % data.shape[i]).encode('ascii')) data = data.astype(np.float32) if dim==2: data.tofile(f) else: np.transpose(data, (2, 0, 1)).tofile(f) def read_kitti_flow(flow_file): flow = cv2.imread(flow_file, cv2.IMREAD_ANYDEPTH|cv2.IMREAD_COLOR) flow = flow[:, :, ::-1].astype(np.float32) flow[:, :, :2] = (flow[:, :, :2] - 2**15) / 64.0 return flow def get_flow_mag(flow): return (flow * flow).sum(1, keepdim=True) def get_occ_mask(flow_fw, flow_bw, scale=0.01, bias=0.5): # output 1 means valid and not occluded flow_fw_warp = warp(flow_fw, flow_bw) flow_bw_warp = warp(flow_bw, flow_fw) flow_fw_diff = flow_fw + flow_bw_warp flow_bw_diff = flow_bw + flow_fw_warp fw_occ_thresh = scale * (get_flow_mag(flow_fw) + get_flow_mag(flow_bw_warp)) + bias bw_occ_thresh = scale * (get_flow_mag(flow_bw) + get_flow_mag(flow_fw_warp)) + bias flow_fw_occ = get_flow_mag(flow_fw_diff) < fw_occ_thresh flow_bw_occ = get_flow_mag(flow_bw_diff) < bw_occ_thresh return flow_fw_occ.float(), flow_bw_occ.float()