anycalib/utils.py

from math import pi

import torch
from torch import Tensor

RAD2DEG = 180 / pi


def fast_small_matmul(a: Tensor, b: Tensor) -> Tensor:
    """Fast batched matrix multiplication for small matrices (size < 8x8).

    Code from Georg Bökman and Horace He:
    https://discuss.pytorch.org/t/multiplying-large-batches-of-small-matrices-fast/201181/2

    Args:
        a: (..., N, M) tensor.
        b: (..., M, P) tensor.

    Returns:
        (..., N, P) matmul result.
    """
    return (a.unsqueeze(-1) * b.unsqueeze(-3)).sum(dim=-2)


def solve_2dweighted_lstsq(
    As: Tensor, bs: Tensor, Ws: Tensor | None = None, mask: Tensor | None = None
) -> tuple[Tensor, Tensor]:
    """Solve linear least-squares by forming the normal equations + LU decomposition.

    This function solves the (possibly weighted or/and masked) linear system of equations
    by forming the normal equations an using LU decomposition. As such, this function is
    fast, at the expense of reduced numerical stability that may be significant for
    ill-conditioned systems.

    Args:
        As: (..., N, 2, D) stacked design matrices.
        bs: (..., N, 2) observations.
        Ws: (..., N, 2, 2) matrix weights for each 2D error.
        mask: (..., N) boolean mask for valid observations.

    Returns:
        (..., D) least-squares solution.
        (...,) integer tensor indicating success. 0 if successful. Otherwise, an
            illegal value was found (<0) or the system is singular (>0).
    """
    WAs = As if Ws is None else fast_small_matmul(Ws, As)
    WAs = WAs if mask is None else WAs * mask[..., None, None]
    AtW = WAs.flatten(-3, -2).transpose(-1, -2)  # (..., D, 2*N)
    AtWA = AtW @ As.flatten(-3, -2)  # (..., D, D)
    AtWb = AtW @ bs.flatten(-2, -1)[..., None]  # (..., D, 1)
    sol, info = torch.linalg.solve_ex(AtWA, AtWb.squeeze(-1))
    sol = sol.nan_to_num(1, 1, 1)
    return sol, info


def solve_2dweighted_lstsq_qr(
    As: Tensor, bs: Tensor, Ws: Tensor | None = None, mask: Tensor | None = None
) -> tuple[Tensor, Tensor]:
    """Solve linear least-squares by with QR decomposition.

    This method is more numerically accurate than solving the system by forming the
    normal equations, at the expense of being slower.

    Args:
        As: (..., N, 2, D) stacked design matrices.
        bs: (..., N, 2) observations.
        Ws: (..., N, 2, 2) matrix weights for each 2D error.
        mask: (..., N) boolean mask for valid observations.

    Returns:
        (..., D) least-squares solution.
        (...,) integer tensor indicating success. 0 if successful. Currently it just
            checks if the solution is finite. TODO: based this also on the residuals.
    """
    if Ws is not None:
        Ws_chol, info = torch.linalg.cholesky_ex(Ws)
        mask_ = ((info == 0) if mask is None else mask & (info == 0)).unsqueeze(-1)
        WAs = fast_small_matmul(Ws_chol, As) * mask_[..., None]
        Wbs = fast_small_matmul(Ws_chol, bs[..., None]).squeeze(-1) * mask_
    else:
        WAs = As if mask is None else As * mask[..., None, None]
        Wbs = bs if mask is None else bs * mask[..., None]
    results = torch.linalg.lstsq(
        WAs.flatten(-3, -2), Wbs.flatten(-2, -1), driver="gels"
    )
    info = torch.where(results.solution.isfinite().all(dim=-1), 0, 1)
    return results.solution, info


def cxcy_and_pix_ar_from_rays(
    im_coords: Tensor, rays: Tensor
) -> tuple[Tensor, Tensor, Tensor]:
    """Estimate the principal point and pixel aspect ratio (f_y/f_x) from a set of rays.

    This function works for (radial) camera models whose projection function has the form:
        u = f_x * r(R, Z, params)*X + c_x
        v = f_y * r(R, Z, params)*Y + c_y
    where r(R, Z) is the function that maps the input ray (X, Y, Z) to its radial
    distance in the sensor plane and R := sqrt(X^2 + Y^2).

    Args:
        im_coords: (..., N, 2) image coordinates.
        rays: (..., N, 3) rays.
        cxcy: (..., 2) principal point.

    Returns:
        (..., 2) principal point.
        (...,) pixel aspect ratio.
        (...,) integer tensor indicating success. 0 if successful.
    """
    # form linear system
    A = rays.new_empty(rays.shape)  # (..., N, 3)
    A[..., 0] = rays[..., 1] * im_coords[..., 0]
    A[..., 1] = -rays[..., 1]
    A[..., 2] = rays[..., 0]
    AtA = A.transpose(-1, -2) @ A  # (..., 3, 3)
    Atb = A.transpose(-1, -2) @ (im_coords[..., 1:] * rays[..., :1])  # (..., 3)
    sol, info = torch.linalg.solve_ex(AtA, Atb.squeeze(-1))
    pix_ar = sol[..., 0]
    cxcy = torch.stack((sol[..., 1] / pix_ar, sol[..., 2]), dim=-1)
    return cxcy, pix_ar, info


def cxcy_from_rays(im_coords: Tensor, rays: Tensor) -> tuple[Tensor, Tensor]:
    """Estimate the principal point from a set of rays. This function assumes that
    pixels are perfect squares, i.e., that pixel aspect ratio (f_y/f_x) = 1.

    This function works for (radial) camera models whose projection function has the form:
        u = f * r(R, Z, params)*X + c_x
        v = f * r(R, Z, params)*Y + c_y
    where r(R, Z, params) is the function that maps the input ray (X, Y, Z) to its radial
    distance in the sensor plane and R := sqrt(X^2 + Y^2).

    Args:
        im_coords: (..., N, 2) image coordinates.
        rays: (..., N, 3) rays.
        cxcy: (..., 2) principal point.

    Returns:
        (..., 2) principal point.
        (...,) integer tensor indicating success. 0 if successful.
    """
    # form linear system
    A = torch.stack((rays[..., 1], -rays[..., 0]), dim=-1)  # (..., N, 2)
    b = (im_coords.flip(-1) * rays[..., :2]).diff(dim=-1)  # (..., N, 1)
    AtA = A.transpose(-1, -2) @ A  # (..., 3, 3)
    Atb = A.transpose(-1, -2) @ b  # (..., 3)
    cxcy_sol, info = torch.linalg.solve_ex(AtA, Atb.squeeze(-1))
    return cxcy_sol, info


def pixel_aspect_ratio_from_rays(
    im_coords: Tensor, rays: Tensor, cxcy: Tensor
) -> Tensor:
    """Compute the pixel aspect ratio f_x/f_y from a set of rays.

    This method estimates the pixel aspect ratio (f_x/f_y) for radial camera models
    whose projection function has the form:
        u = f * r(R, Z, params)*X + c_x
        v = f * r(R, Z, params)*Y + c_y
    where r(R, Z) is the function that maps the input ray (X, Y, Z) to its radial
    distance in the sensor plane and R := sqrt(X^2 + Y^2).

    Args:
        im_coords: (..., N, 2) image coordinates.
        rays: (..., N, 3) rays.
        cxcy: (..., 2) principal point.

    Returns:
        (...,) pixel aspect ratio.
    """
    # fx/fy = (u-cx)*Y / (v-cy)*X
    num = torch.abs((im_coords[..., 0] - cxcy[..., 0]) * rays[..., 1])  # (..., N)
    den = torch.abs((im_coords[..., 1] - cxcy[..., 1]) * rays[..., 0])
    # mask out rays with X=0 or Y=0 and image coordinates with u-c_x = 0 or v-c_y = 0
    eps = torch.finfo(num.dtype).eps
    mask = (num > eps) & (den > eps)
    num = torch.where(mask, num, 1)
    den = torch.where(mask, den, 1)
    pixel_ar = masked_mean(num / den, mask, dim=-1)
    return pixel_ar


def masked_mean(a: Tensor, mask: Tensor, dim: int | tuple[int, ...]) -> Tensor:
    """Compute the mean of a tensor along a dimension, ignoring masked values.

    Args:
        a: Tensor to compute the mean.
        mask: Boolean mask. Must be broadcastable to a.shape.
        dim: Dimension along which to compute the mean.

    Returns:
        Mean of the tensor.
    """
    return (a * mask).sum(dim) / mask.sum(dim).clamp(min=1)