utils/camera_view_utils.py

import os
import json
import numpy as np
import torch
from scene.cameras import Camera as GSCamera
from utils.graphics_utils import focal2fov


def generate_camera_rotation_matrix(camera_to_object, object_vertical_downward):
    camera_to_object = camera_to_object / np.linalg.norm(
        camera_to_object
    )  # last column
    # the second column of rotation matrix is pointing toward the downward vertical direction
    camera_y = (
        object_vertical_downward
        - np.dot(object_vertical_downward, camera_to_object) * camera_to_object
    )
    camera_y = camera_y / np.linalg.norm(camera_y)  # second column
    first_column = np.cross(camera_y, camera_to_object)
    R = np.column_stack((first_column, camera_y, camera_to_object))
    return R


# supply vertical vector in world space
def generate_local_coord(vertical_vector):
    vertical_vector = vertical_vector / np.linalg.norm(vertical_vector)
    horizontal_1 = np.array([1, 1, 1])
    if np.abs(np.dot(horizontal_1, vertical_vector)) < 0.01:
        horizontal_1 = np.array([0.72, 0.37, -0.67])
    # gram schimit
    horizontal_1 = (
        horizontal_1 - np.dot(horizontal_1, vertical_vector) * vertical_vector
    )
    horizontal_1 = horizontal_1 / np.linalg.norm(horizontal_1)
    horizontal_2 = np.cross(horizontal_1, vertical_vector)

    return vertical_vector, horizontal_1, horizontal_2


# scalar (in degrees), scalar (in degrees), scalar, vec3, mat33 = [horizontal_1; horizontal_2; vertical];  -> vec3
def get_point_on_sphere(azimuth, elevation, radius, center, observant_coordinates):
    canonical_coordinates = (
        np.array(
            [
                np.cos(azimuth / 180.0 * np.pi) * np.cos(elevation / 180.0 * np.pi),
                np.sin(azimuth / 180.0 * np.pi) * np.cos(elevation / 180.0 * np.pi),
                np.sin(elevation / 180.0 * np.pi),
            ]
        )
        * radius
    )

    return center + observant_coordinates @ canonical_coordinates


def get_camera_position_and_rotation(
    azimuth, elevation, radius, view_center, observant_coordinates
):
    # get camera position
    position = get_point_on_sphere(
        azimuth, elevation, radius, view_center, observant_coordinates
    )
    # get rotation matrix
    R = generate_camera_rotation_matrix(
        view_center - position, -observant_coordinates[:, 2]
    )
    return position, R


def get_current_radius_azimuth_and_elevation(
    camera_position, view_center, observesant_coordinates
):
    center2camera = -view_center + camera_position
    radius = np.linalg.norm(center2camera)
    dot_product = np.dot(center2camera, observesant_coordinates[:, 2])
    cosine = dot_product / (
        np.linalg.norm(center2camera) * np.linalg.norm(observesant_coordinates[:, 2])
    )
    elevation = np.rad2deg(np.pi / 2.0 - np.arccos(cosine))
    proj_onto_hori = center2camera - dot_product * observesant_coordinates[:, 2]
    dot_product2 = np.dot(proj_onto_hori, observesant_coordinates[:, 0])
    cosine2 = dot_product2 / (
        np.linalg.norm(proj_onto_hori) * np.linalg.norm(observesant_coordinates[:, 0])
    )

    if np.dot(proj_onto_hori, observesant_coordinates[:, 1]) > 0:
        azimuth = np.rad2deg(np.arccos(cosine2))
    else:
        azimuth = -np.rad2deg(np.arccos(cosine2))
    return radius, azimuth, elevation


def get_camera_view(
    model_path,
    default_camera_index=0,
    center_view_world_space=None,
    observant_coordinates=None,
    show_hint=False,
    init_azimuthm=None,
    init_elevation=None,
    init_radius=None,
    move_camera=False,
    current_frame=0,
    delta_a=0,
    delta_e=0,
    delta_r=0,
    downsample=1.0,
):
    """Load one of the default cameras for the scene."""
    cam_path = os.path.join(model_path, "cameras.json")
    with open(cam_path) as f:
        data = json.load(f)

        camera_view_info = {}
        camera_view_info["init_azimuthm"] = init_azimuthm
        camera_view_info["init_elevation"] = init_elevation
        camera_view_info["init_radius"] = init_radius

        if show_hint:
            if default_camera_index < 0:
                default_camera_index = 0
            r, a, e = get_current_radius_azimuth_and_elevation(
                data[default_camera_index]["position"],
                center_view_world_space,
                observant_coordinates,
            )
            print("Default camera ", default_camera_index, " has")
            print("azimuth:    ", a)
            print("elevation:  ", e)
            print("radius:     ", r)
            print("Now exit program and set your own input!")
            exit()

        if default_camera_index > -1:
            raw_camera = data[default_camera_index]
            # print(raw_camera)

            r, a, e = get_current_radius_azimuth_and_elevation(
                raw_camera["position"],
                center_view_world_space,
                observant_coordinates,
            )
            # if raw_camera["width"] < raw_camera["height"]:
            #     a = -a
            camera_view_info["init_azimuthm"] = a
            camera_view_info["init_elevation"] = e
            camera_view_info["init_radius"] = r

            init_azimuthm = a
            init_elevation = e
            init_radius = r

            if move_camera:
                assert delta_a is not None
                assert delta_e is not None
                assert delta_r is not None
                position, R = get_camera_position_and_rotation(
                    init_azimuthm + current_frame * delta_a,
                    init_elevation + current_frame * delta_e,
                    init_radius + current_frame * delta_r,
                    center_view_world_space,
                    observant_coordinates,
                )
                # print("position", position)
                raw_camera["rotation"] = R.tolist()
                raw_camera["position"] = position.tolist()

            # print("Default camera ", default_camera_index, " has")
            # print("azimuth:    ", a)
            # print("elevation:  ", e)
            # print("radius:     ", r)
        else:
            raw_camera = data[0]

            r, a, e = get_current_radius_azimuth_and_elevation(
                data[default_camera_index]["position"],
                center_view_world_space,
                observant_coordinates,
            )

            if move_camera:
                assert delta_a is not None
                assert delta_e is not None
                assert delta_r is not None
                position, R = get_camera_position_and_rotation(
                    init_azimuthm + current_frame * delta_a,
                    init_elevation + current_frame * delta_e,
                    init_radius + current_frame * delta_r,
                    center_view_world_space,
                    observant_coordinates,
                )
            else:
                position, R = get_camera_position_and_rotation(
                    init_azimuthm,
                    init_elevation,
                    init_radius,
                    center_view_world_space,
                    observant_coordinates,
                )
            raw_camera["rotation"] = R.tolist()
            raw_camera["position"] = position.tolist()


        tmp = np.zeros((4, 4))
        tmp[:3, :3] = raw_camera["rotation"]
        tmp[:3, 3] = raw_camera["position"]
        tmp[3, 3] = 1
        C2W = np.linalg.inv(tmp)
        R = C2W[:3, :3].transpose()
        T = C2W[:3, 3]
        # if 'pac_nerf/toothpaste' in model_path:
        #     T = C2W[:3, 3] * 1.5

        width = min(raw_camera["width"], 1920)
        height = min(raw_camera["height"], 1920)
        # fovx = focal2fov(raw_camera["fx"], width)
        # fovy = focal2fov(raw_camera["fy"], height)
        width = int(width * downsample)
        height = int(height * downsample)

        fovx = focal2fov(raw_camera["fx"]*downsample, width)
        fovy = focal2fov(raw_camera["fy"]*downsample, height)


        return GSCamera(
            colmap_id=0,
            R=R,
            T=T,
            FoVx=fovx,
            FoVy=fovy,
            image_width=width,
            image_height=height,
            image=torch.zeros((3, height, width)),  # fake
            gt_alpha_mask=None,
            image_name="fake",
            image_path="fake",
            uid=0,
            preload_img=False
        ), camera_view_info

def get_camera_view_endonerf():
    # endonerf camera para
    R_for_GSCamera = np.array([[1., -0., -0.],
                        [0., -1., -0.],
                        [0., -0., -1.]], dtype=float)
    T_for_GSCamera = np.zeros(3, dtype=float)
    # T_for_GSCamera[2] +=10 # 临时生成器械效果图
    # T_for_GSCamera[1] +=5 # 临时生成器械效果图

    FoVx = 1.0239093368021417
    FoVy = 0.8449463442193673
    width = int(640)
    height = int(512)
    image_tensor = torch.zeros((3, height, width))  # fake
    current_camera =  GSCamera(
        colmap_id=0,
        R=R_for_GSCamera,
        T=T_for_GSCamera,
        FoVx=FoVx,
        FoVy=FoVy,
        image_width=width,
        image_height=height,
        image=image_tensor,
        gt_alpha_mask=None,
        image_name="fake",
        image_path="fake",
        uid=0,
        preload_img=False
    )
    return current_camera

def get_camera_view_stereomis():
    # endonerf camera para
    R_for_GSCamera = np.array([[1., -0., -0.],
                        [0., -1., -0.],
                        [0., -0., -1.]], dtype=float)
    T_for_GSCamera = np.zeros(3, dtype=float)
    FoVx = 1.1036297361167346
    FoVy = 0.9152335928441784
    width = int(640)
    height = int(512)
    image_tensor = torch.zeros((3, height, width))  # fake
    current_camera =  GSCamera(
        colmap_id=0,
        R=R_for_GSCamera,
        T=T_for_GSCamera,
        FoVx=FoVx,
        FoVy=FoVy,
        image_width=width,
        image_height=height,
        image=image_tensor,
        gt_alpha_mask=None,
        image_name="fake",
        image_path="fake",
        uid=0,
        preload_img=False
    )
    return current_camera



def get_camera_view_cholecseg_sub():
    # endonerf camera para
    R_for_GSCamera = np.array([[1., -0., -0.],
                        [0., 1., -0.],
                        [0., -0., 1.]], dtype=float)
    T_for_GSCamera = np.zeros(3, dtype=float)
    FoVx = 0.7483519078147721
    FoVy = 0.4344497977059072
    width = int(854)
    height = int(480)
    image_tensor = torch.zeros((3, height, width))  # fake
    current_camera =  GSCamera(
        colmap_id=0,
        R=R_for_GSCamera,
        T=T_for_GSCamera,
        FoVx=FoVx,
        FoVy=FoVy,
        image_width=width,
        image_height=height,
        image=image_tensor,
        gt_alpha_mask=None,
        image_name="fake",
        image_path="fake",
        uid=0,
        preload_img=False
    )
    return current_camera


def get_camera_view_porcine_endo():
    # endonerf camera para
    R_for_GSCamera = np.array([[1., -0., -0.],
                        [0., 1., -0.],
                        [0., -0., 1.]], dtype=float)
    T_for_GSCamera = np.zeros(3, dtype=float)
    # T_for_GSCamera[2] +=10 # 临时生成器械效果图
    FoVx = 0.8283241463683268
    FoVy = 0.6349340651524208
    width = int(640)
    height = int(480)
    image_tensor = torch.zeros((3, height, width))  # fake
    current_camera =  GSCamera(
        colmap_id=0,
        R=R_for_GSCamera,
        T=T_for_GSCamera,
        FoVx=FoVx,
        FoVy=FoVy,
        image_width=width,
        image_height=height,
        image=image_tensor,
        gt_alpha_mask=None,
        image_name="fake",
        image_path="fake",
        uid=0,
        preload_img=False
    )
    return current_camera