inference/v2v_data/DepthCrafter/visualization/visualization_pcd.py

"""Record3D visualizer
"""

import time
from decord import VideoReader, cpu

import numpy as np
import tyro
import viser
import viser.extras
import viser.transforms as tf
from tqdm.auto import tqdm


def main(
    data_path: str,
    vid_name: str,
    downsample_factor: int = 8,
    max_frames: int = 100,
    share: bool = False,
    point_size=0.01,
) -> None:

    server = viser.ViserServer()
    if share:
        server.request_share_url()

    print("Loading frames!")
    dis_path = data_path + "/" + vid_name + ".npz"
    vid_path = data_path + "/" + vid_name + "_input.mp4"

    disp_map = np.load(dis_path)["depth"][:, :, :]
    T = disp_map.shape[0]
    H = disp_map.shape[1]
    W = disp_map.shape[2]

    disp_max = disp_map.max()
    disp_min = disp_map.min()
    disp_map = (disp_map - disp_min) / (disp_max - disp_min)

    vr = VideoReader(vid_path, ctx=cpu(0))
    vid = vr[:].asnumpy()[:, 0:H, 0:W]
    fps = vr.get_avg_fps()
    num_frames = min(max_frames, T)

    # Add playback UI.
    with server.gui.add_folder("Playback"):
        gui_timestep = server.gui.add_slider(
            "Timestep",
            min=0,
            max=num_frames - 1,
            step=1,
            initial_value=0,
            disabled=True,
        )
        gui_next_frame = server.gui.add_button("Next Frame", disabled=True)
        gui_prev_frame = server.gui.add_button("Prev Frame", disabled=True)
        gui_playing = server.gui.add_checkbox("Playing", True)
        gui_framerate = server.gui.add_slider(
            "FPS", min=1, max=60, step=0.1, initial_value=fps
        )
        gui_framerate_options = server.gui.add_button_group(
            "FPS options", ("10", "20", "30", "60")
        )

    # Frame step buttons.
    @gui_next_frame.on_click
    def _(_) -> None:
        gui_timestep.value = (gui_timestep.value + 1) % num_frames

    @gui_prev_frame.on_click
    def _(_) -> None:
        gui_timestep.value = (gui_timestep.value - 1) % num_frames

    # Disable frame controls when we're playing.
    @gui_playing.on_update
    def _(_) -> None:
        gui_timestep.disabled = gui_playing.value
        gui_next_frame.disabled = gui_playing.value
        gui_prev_frame.disabled = gui_playing.value

    # Set the framerate when we click one of the options.
    @gui_framerate_options.on_click
    def _(_) -> None:
        gui_framerate.value = int(gui_framerate_options.value)

    prev_timestep = gui_timestep.value

    # Toggle frame visibility when the timestep slider changes.
    @gui_timestep.on_update
    def _(_) -> None:
        nonlocal prev_timestep
        current_timestep = gui_timestep.value
        with server.atomic():
            frame_nodes[current_timestep].visible = True
            frame_nodes[prev_timestep].visible = False
        prev_timestep = current_timestep
        server.flush()  # Optional!

    # Load in frames.
    server.scene.add_frame(
        "/frames",
        wxyz=tf.SO3.exp(np.array([0.0, 0.0, 0.0])).wxyz,
        position=(0, 0, 0),
        show_axes=False,
    )
    frame_nodes: list[viser.FrameHandle] = []
    for i in tqdm(range(num_frames)):

        # Add base frame.
        frame_nodes.append(server.scene.add_frame(f"/frames/t{i}", show_axes=False))

        position_image = np.where(np.zeros([H, W]) == 0)
        v = np.array(position_image[0])
        u = np.array(position_image[1])
        d = disp_map[i, v, u]

        zc = 1.0 / (d + 0.1)
        # zc = 1.0 / (d + 1e-8)

        xc = zc * (u - (W / 2.0)) / (W / 2.0)
        yc = zc * (v - (H / 2.0)) / (H / 2.0)

        zc -= 4  # disp_max * 0.2

        points = np.stack((xc, yc, zc), axis=1)
        colors = vid[i, v, u]

        points = points[::downsample_factor]
        colors = colors[::downsample_factor]

        # Place the point cloud in the frame.
        server.scene.add_point_cloud(
            name=f"/frames/t{i}/point_cloud",
            points=points,
            colors=colors,
            point_size=point_size,  # 0.007,
            point_shape="rounded",
        )

    # Hide all but the current frame.
    for i, frame_node in enumerate(frame_nodes):
        frame_node.visible = i == gui_timestep.value

    # Playback update loop.
    prev_timestep = gui_timestep.value
    while True:
        if gui_playing.value:
            gui_timestep.value = (gui_timestep.value + 1) % num_frames

        time.sleep(1.0 / gui_framerate.value)


if __name__ == "__main__":
    tyro.cli(
        main(
            # dir path of saved rgb.mp4 and disp.npz, modify it to your own dir
            data_path="./demo_output",
            # sample name, modify it to your own sample name
            vid_name="example_01",
            # downsample factor of dense pcd
            downsample_factor=8,
            # point cloud size
            point_size=0.007,
        )
    )