scope/geometry.py

"""Camera-ray and Plücker-coordinate utilities used by SCoPE."""

from __future__ import annotations

import torch
import torch.nn.functional as F


def compute_camera_rays(
    c2w: torch.Tensor,
    intrinsics: torch.Tensor,
    patches_y: int,
    patches_x: int,
    image_height: int,
    image_width: int,
) -> tuple[torch.Tensor, torch.Tensor]:
    """Return world-space ray origins and directions on the token grid.

    Args:
        c2w: Camera-to-world matrices, shaped ``[B, T, 4, 4]``.
        intrinsics: Pinhole intrinsics, shaped ``[B, T, 3, 3]``.
    """
    batch, frames = c2w.shape[:2]
    dtype = c2w.dtype
    device = c2w.device
    patch_width = image_width / patches_x
    patch_height = image_height / patches_y
    u = torch.linspace(
        0.5 * patch_width,
        image_width - 0.5 * patch_width,
        patches_x,
        device=device,
        dtype=dtype,
    )
    v = torch.linspace(
        0.5 * patch_height,
        image_height - 0.5 * patch_height,
        patches_y,
        device=device,
        dtype=dtype,
    )
    grid_u, grid_v = torch.meshgrid(u, v, indexing="xy")
    grid_u = grid_u[None, None].expand(batch, frames, -1, -1)
    grid_v = grid_v[None, None].expand(batch, frames, -1, -1)

    fx = intrinsics[..., 0, 0, None, None]
    fy = intrinsics[..., 1, 1, None, None]
    cx = intrinsics[..., 0, 2, None, None]
    cy = intrinsics[..., 1, 2, None, None]
    directions_camera = torch.stack(
        ((grid_u - cx) / fx, (grid_v - cy) / fy, torch.ones_like(grid_u)), dim=-1
    )  # [B, T, H, W, 3]
    directions_camera = F.normalize(directions_camera, dim=-1, eps=1e-8)

    rotation = c2w[..., :3, :3]
    translation = c2w[..., :3, 3]
    directions_world = torch.einsum("btij,bthwj->bthwi", rotation, directions_camera)
    directions_world = F.normalize(directions_world, dim=-1, eps=1e-8)
    origins_world = translation[..., None, None, :].expand_as(directions_world)
    return origins_world, directions_world


def compute_plucker_coordinates(
    c2w: torch.Tensor,
    intrinsics: torch.Tensor,
    patches_y: int,
    patches_x: int,
    image_height: int,
    image_width: int,
) -> torch.Tensor:
    """Return ``(direction, moment)`` coordinates shaped ``[B, S, 6]``."""
    origins, directions = compute_camera_rays(
        c2w,
        intrinsics,
        patches_y,
        patches_x,
        image_height,
        image_width,
    )
    moments = torch.cross(origins, directions, dim=-1)
    batch = c2w.shape[0]
    return torch.cat((directions.reshape(batch, -1, 3), moments.reshape(batch, -1, 3)), dim=-1)