import os import json import numpy as np import torch from scene.cameras import Camera as GSCamera from utils.graphics_utils import focal2fov def generate_camera_rotation_matrix(camera_to_object, object_vertical_downward): camera_to_object = camera_to_object / np.linalg.norm( camera_to_object ) # last column # the second column of rotation matrix is pointing toward the downward vertical direction camera_y = ( object_vertical_downward - np.dot(object_vertical_downward, camera_to_object) * camera_to_object ) camera_y = camera_y / np.linalg.norm(camera_y) # second column first_column = np.cross(camera_y, camera_to_object) R = np.column_stack((first_column, camera_y, camera_to_object)) return R # supply vertical vector in world space def generate_local_coord(vertical_vector): vertical_vector = vertical_vector / np.linalg.norm(vertical_vector) horizontal_1 = np.array([1, 1, 1]) if np.abs(np.dot(horizontal_1, vertical_vector)) < 0.01: horizontal_1 = np.array([0.72, 0.37, -0.67]) # gram schimit horizontal_1 = ( horizontal_1 - np.dot(horizontal_1, vertical_vector) * vertical_vector ) horizontal_1 = horizontal_1 / np.linalg.norm(horizontal_1) horizontal_2 = np.cross(horizontal_1, vertical_vector) return vertical_vector, horizontal_1, horizontal_2 # scalar (in degrees), scalar (in degrees), scalar, vec3, mat33 = [horizontal_1; horizontal_2; vertical]; -> vec3 def get_point_on_sphere(azimuth, elevation, radius, center, observant_coordinates): canonical_coordinates = ( np.array( [ np.cos(azimuth / 180.0 * np.pi) * np.cos(elevation / 180.0 * np.pi), np.sin(azimuth / 180.0 * np.pi) * np.cos(elevation / 180.0 * np.pi), np.sin(elevation / 180.0 * np.pi), ] ) * radius ) return center + observant_coordinates @ canonical_coordinates def get_camera_position_and_rotation( azimuth, elevation, radius, view_center, observant_coordinates ): # get camera position position = get_point_on_sphere( azimuth, elevation, radius, view_center, observant_coordinates ) # get rotation matrix R = generate_camera_rotation_matrix( view_center - position, -observant_coordinates[:, 2] ) return position, R def get_current_radius_azimuth_and_elevation( camera_position, view_center, observesant_coordinates ): center2camera = -view_center + camera_position radius = np.linalg.norm(center2camera) dot_product = np.dot(center2camera, observesant_coordinates[:, 2]) cosine = dot_product / ( np.linalg.norm(center2camera) * np.linalg.norm(observesant_coordinates[:, 2]) ) elevation = np.rad2deg(np.pi / 2.0 - np.arccos(cosine)) proj_onto_hori = center2camera - dot_product * observesant_coordinates[:, 2] dot_product2 = np.dot(proj_onto_hori, observesant_coordinates[:, 0]) cosine2 = dot_product2 / ( np.linalg.norm(proj_onto_hori) * np.linalg.norm(observesant_coordinates[:, 0]) ) if np.dot(proj_onto_hori, observesant_coordinates[:, 1]) > 0: azimuth = np.rad2deg(np.arccos(cosine2)) else: azimuth = -np.rad2deg(np.arccos(cosine2)) return radius, azimuth, elevation def get_camera_view( model_path, default_camera_index=0, center_view_world_space=None, observant_coordinates=None, show_hint=False, init_azimuthm=None, init_elevation=None, init_radius=None, move_camera=False, current_frame=0, delta_a=0, delta_e=0, delta_r=0, downsample=1.0, ): """Load one of the default cameras for the scene.""" cam_path = os.path.join(model_path, "cameras.json") with open(cam_path) as f: data = json.load(f) camera_view_info = {} camera_view_info["init_azimuthm"] = init_azimuthm camera_view_info["init_elevation"] = init_elevation camera_view_info["init_radius"] = init_radius if show_hint: if default_camera_index < 0: default_camera_index = 0 r, a, e = get_current_radius_azimuth_and_elevation( data[default_camera_index]["position"], center_view_world_space, observant_coordinates, ) print("Default camera ", default_camera_index, " has") print("azimuth: ", a) print("elevation: ", e) print("radius: ", r) print("Now exit program and set your own input!") exit() if default_camera_index > -1: raw_camera = data[default_camera_index] # print(raw_camera) r, a, e = get_current_radius_azimuth_and_elevation( raw_camera["position"], center_view_world_space, observant_coordinates, ) # if raw_camera["width"] < raw_camera["height"]: # a = -a camera_view_info["init_azimuthm"] = a camera_view_info["init_elevation"] = e camera_view_info["init_radius"] = r init_azimuthm = a init_elevation = e init_radius = r if move_camera: assert delta_a is not None assert delta_e is not None assert delta_r is not None position, R = get_camera_position_and_rotation( init_azimuthm + current_frame * delta_a, init_elevation + current_frame * delta_e, init_radius + current_frame * delta_r, center_view_world_space, observant_coordinates, ) # print("position", position) raw_camera["rotation"] = R.tolist() raw_camera["position"] = position.tolist() # print("Default camera ", default_camera_index, " has") # print("azimuth: ", a) # print("elevation: ", e) # print("radius: ", r) else: raw_camera = data[0] r, a, e = get_current_radius_azimuth_and_elevation( data[default_camera_index]["position"], center_view_world_space, observant_coordinates, ) if move_camera: assert delta_a is not None assert delta_e is not None assert delta_r is not None position, R = get_camera_position_and_rotation( init_azimuthm + current_frame * delta_a, init_elevation + current_frame * delta_e, init_radius + current_frame * delta_r, center_view_world_space, observant_coordinates, ) else: position, R = get_camera_position_and_rotation( init_azimuthm, init_elevation, init_radius, center_view_world_space, observant_coordinates, ) raw_camera["rotation"] = R.tolist() raw_camera["position"] = position.tolist() tmp = np.zeros((4, 4)) tmp[:3, :3] = raw_camera["rotation"] tmp[:3, 3] = raw_camera["position"] tmp[3, 3] = 1 C2W = np.linalg.inv(tmp) R = C2W[:3, :3].transpose() T = C2W[:3, 3] # if 'pac_nerf/toothpaste' in model_path: # T = C2W[:3, 3] * 1.5 width = min(raw_camera["width"], 1920) height = min(raw_camera["height"], 1920) # fovx = focal2fov(raw_camera["fx"], width) # fovy = focal2fov(raw_camera["fy"], height) width = int(width * downsample) height = int(height * downsample) fovx = focal2fov(raw_camera["fx"]*downsample, width) fovy = focal2fov(raw_camera["fy"]*downsample, height) return GSCamera( colmap_id=0, R=R, T=T, FoVx=fovx, FoVy=fovy, image_width=width, image_height=height, image=torch.zeros((3, height, width)), # fake gt_alpha_mask=None, image_name="fake", image_path="fake", uid=0, preload_img=False ), camera_view_info def get_camera_view_endonerf(): # endonerf camera para R_for_GSCamera = np.array([[1., -0., -0.], [0., -1., -0.], [0., -0., -1.]], dtype=float) T_for_GSCamera = np.zeros(3, dtype=float) # T_for_GSCamera[2] +=10 # 临时生成器械效果图 # T_for_GSCamera[1] +=5 # 临时生成器械效果图 FoVx = 1.0239093368021417 FoVy = 0.8449463442193673 width = int(640) height = int(512) image_tensor = torch.zeros((3, height, width)) # fake current_camera = GSCamera( colmap_id=0, R=R_for_GSCamera, T=T_for_GSCamera, FoVx=FoVx, FoVy=FoVy, image_width=width, image_height=height, image=image_tensor, gt_alpha_mask=None, image_name="fake", image_path="fake", uid=0, preload_img=False ) return current_camera def get_camera_view_stereomis(): # endonerf camera para R_for_GSCamera = np.array([[1., -0., -0.], [0., -1., -0.], [0., -0., -1.]], dtype=float) T_for_GSCamera = np.zeros(3, dtype=float) FoVx = 1.1036297361167346 FoVy = 0.9152335928441784 width = int(640) height = int(512) image_tensor = torch.zeros((3, height, width)) # fake current_camera = GSCamera( colmap_id=0, R=R_for_GSCamera, T=T_for_GSCamera, FoVx=FoVx, FoVy=FoVy, image_width=width, image_height=height, image=image_tensor, gt_alpha_mask=None, image_name="fake", image_path="fake", uid=0, preload_img=False ) return current_camera def get_camera_view_cholecseg_sub(): # endonerf camera para R_for_GSCamera = np.array([[1., -0., -0.], [0., 1., -0.], [0., -0., 1.]], dtype=float) T_for_GSCamera = np.zeros(3, dtype=float) FoVx = 0.7483519078147721 FoVy = 0.4344497977059072 width = int(854) height = int(480) image_tensor = torch.zeros((3, height, width)) # fake current_camera = GSCamera( colmap_id=0, R=R_for_GSCamera, T=T_for_GSCamera, FoVx=FoVx, FoVy=FoVy, image_width=width, image_height=height, image=image_tensor, gt_alpha_mask=None, image_name="fake", image_path="fake", uid=0, preload_img=False ) return current_camera def get_camera_view_porcine_endo(): # endonerf camera para R_for_GSCamera = np.array([[1., -0., -0.], [0., 1., -0.], [0., -0., 1.]], dtype=float) T_for_GSCamera = np.zeros(3, dtype=float) # T_for_GSCamera[2] +=10 # 临时生成器械效果图 FoVx = 0.8283241463683268 FoVy = 0.6349340651524208 width = int(640) height = int(480) image_tensor = torch.zeros((3, height, width)) # fake current_camera = GSCamera( colmap_id=0, R=R_for_GSCamera, T=T_for_GSCamera, FoVx=FoVx, FoVy=FoVy, image_width=width, image_height=height, image=image_tensor, gt_alpha_mask=None, image_name="fake", image_path="fake", uid=0, preload_img=False ) return current_camera