mia/fpv/utils_sfm.py

# Copied from OrienterNet
# Copyright (c) Meta Platforms, Inc. and affiliates.

from typing import List, Tuple

import cv2
import numpy as np
from opensfm import features
from opensfm.pygeometry import Camera, compute_camera_mapping, Pose
from opensfm.pymap import Shot
from scipy.spatial.transform import Rotation


def keyframe_selection(shots: List[Shot], min_dist: float = 4) -> List[int]:
    camera_centers = np.stack([shot.pose.get_origin() for shot in shots], 0)
    distances = np.linalg.norm(np.diff(camera_centers, axis=0), axis=1)
    selected = [0]
    cum = 0
    for i in range(1, len(camera_centers)):
        cum += distances[i - 1]
        if cum >= min_dist:
            selected.append(i)
            cum = 0
    return selected


def perspective_camera_from_pano(camera: Camera, size: int) -> Camera:
    camera_new = Camera.create_perspective(0.5, 0, 0)
    camera_new.height = camera_new.width = size
    camera_new.id = "perspective_from_pano"
    return camera_new


def scale_camera(camera: Camera, max_size: int) -> Camera:
    height = camera.height
    width = camera.width
    factor = max_size / float(max(height, width))
    if factor >= 1:
        return camera
    camera.width = int(round(width * factor))
    camera.height = int(round(height * factor))
    return camera


class PanoramaUndistorter:
    def __init__(self, camera_pano: Camera, camera_new: Camera):
        w, h = camera_new.width, camera_new.height
        self.shape = (h, w)

        dst_y, dst_x = np.indices(self.shape).astype(np.float32)
        dst_pixels_denormalized = np.column_stack([dst_x.ravel(), dst_y.ravel()])
        dst_pixels = features.normalized_image_coordinates(
            dst_pixels_denormalized, w, h
        )
        self.dst_bearings = camera_new.pixel_bearing_many(dst_pixels)

        self.camera_pano = camera_pano
        self.camera_perspective = camera_new

    def __call__(
        self, image: np.ndarray, panoshot: Shot, perspectiveshot: Shot
    ) -> np.ndarray:
        # Rotate to panorama reference frame
        rotation = np.dot(
            panoshot.pose.get_rotation_matrix(),
            perspectiveshot.pose.get_rotation_matrix().T,
        )
        rotated_bearings = np.dot(self.dst_bearings, rotation.T)

        # Project to panorama pixels
        src_pixels = panoshot.camera.project_many(rotated_bearings)
        src_pixels_denormalized = features.denormalized_image_coordinates(
            src_pixels, image.shape[1], image.shape[0]
        )
        src_pixels_denormalized.shape = self.shape + (2,)

        # Sample color
        x = src_pixels_denormalized[..., 0].astype(np.float32)
        y = src_pixels_denormalized[..., 1].astype(np.float32)
        colors = cv2.remap(image, x, y, cv2.INTER_LINEAR, borderMode=cv2.BORDER_WRAP)
        return colors


class CameraUndistorter:
    def __init__(self, camera_distorted: Camera, camera_new: Camera):
        self.maps = compute_camera_mapping(
            camera_distorted,
            camera_new,
            camera_distorted.width,
            camera_distorted.height,
        )
        self.camera_perspective = camera_new
        self.camera_distorted = camera_distorted

    def __call__(self, image: np.ndarray) -> np.ndarray:
        assert image.shape[:2] == (
            self.camera_distorted.height,
            self.camera_distorted.width,
        )
        undistorted = cv2.remap(image, *self.maps, cv2.INTER_LINEAR)
        resized = cv2.resize(
            undistorted,
            (self.camera_perspective.width, self.camera_perspective.height),
            interpolation=cv2.INTER_AREA,
        )
        return resized


def render_panorama(
    shot: Shot,
    pano: np.ndarray,
    undistorter: PanoramaUndistorter,
    offset: float = 0.0,
) -> Tuple[List[Shot], List[np.ndarray]]:
    yaws = [0, 90, 180, 270]
    suffixes = ["front", "left", "back", "right"]
    images = []
    shots = []

    # To reduce aliasing, since cv2.remap does not support area samplimg,
    # we first resize with anti-aliasing.
    h, w = undistorter.shape
    h, w = (w * 2, w * 4)  # assuming 90deg FOV
    pano_resized = cv2.resize(pano, (w, h), interpolation=cv2.INTER_AREA)

    for yaw, suffix in zip(yaws, suffixes):
        R_pano2persp = Rotation.from_euler("Y", yaw + offset, degrees=True).as_matrix()
        name = f"{shot.id}_{suffix}"
        shot_new = Shot(
            name,
            undistorter.camera_perspective,
            Pose.compose(Pose(R_pano2persp), shot.pose),
        )
        shot_new.metadata = shot.metadata
        perspective = undistorter(pano_resized, shot, shot_new)
        images.append(perspective)
        shots.append(shot_new)
    return shots, images


def undistort_camera(
    shot: Shot, image: np.ndarray, undistorter: CameraUndistorter
) -> Tuple[Shot, np.ndarray]:
    name = f"{shot.id}_undistorted"
    shot_out = Shot(name, undistorter.camera_perspective, shot.pose)
    shot_out.metadata = shot.metadata
    undistorted = undistorter(image)
    return shot_out, undistorted


def undistort_shot(
    image_raw: np.ndarray,
    shot_orig: Shot,
    undistorter,
    pano_offset: float,
) -> Tuple[List[Shot], List[np.ndarray]]:
    camera = shot_orig.camera
    if image_raw.shape[:2] != (camera.height, camera.width):
        raise ValueError(
            shot_orig.id, image_raw.shape[:2], (camera.height, camera.width)
        )
    if camera.is_panorama(camera.projection_type):
        shots, undistorted = render_panorama(
            shot_orig, image_raw, undistorter, offset=pano_offset
        )
    elif camera.projection_type in ("perspective", "fisheye"):
        shot, undistorted = undistort_camera(shot_orig, image_raw, undistorter)
        shots, undistorted = [shot], [undistorted]
    else:
        raise NotImplementedError(camera.projection_type)
    return shots, undistorted