anycalib/visualization/viz_2d.py

import operator
from collections.abc import Callable
from math import pi

import matplotlib.patheffects as pe
import numpy as np
import torch
from matplotlib.axes import Axes
from matplotlib.cm import ScalarMappable
from matplotlib.colors import Colormap, LinearSegmentedColormap, Normalize
from matplotlib.typing import ColorType
from numpy.typing import ArrayLike
from torch import Tensor

from anycalib.manifolds import Unit3

RAD2DEG = 180 / pi


def do_detach_cpu(*args):
    return [arg.detach().cpu() if isinstance(arg, Tensor) else arg for arg in args]


def polar_angles_from_rays(rays: Tensor, as_deg: bool = True) -> Tensor:
    """Compute the polar angles of a set of rays.

    Args:
        rays: (..., N, 3) rays.

    Returns:
        (..., N) polar angles.
    """
    r = torch.linalg.norm(rays[..., :2], dim=-1)
    polar_angles = torch.atan2(r, rays[..., 2])
    if as_deg:
        polar_angles *= RAD2DEG
    return polar_angles


def add_colorbar_to_axes(
    ax: Axes,
    sm: ScalarMappable,
    bbox: tuple[float, float, float, float] | None = None,
    label: str | None = None,
    extreme_ticks_only: bool = True,
    **kwargs,
):
    """Add a colorbar to axis.

    Args:
        ax: The axis to which the colorbar should be added.
        sm: The ScalarMappable object to be used for the colorbar.
        bbox: The bounding box of the colorbar in the axis coordinates,
            [x0, y0, width, height], with (x0, y0) specifying the lower-left corner of
            inset Axes where the colorbar is placed, and its width and height. If None,
            defaults to (1.03, 0, 0.04, 1.0).
        label: The label of the colorbar.
        extreme_ticks: If True, the colorbar will have ticks at the extreme values of
            the ScalarMappable `sm`.

    Returns:
        The colorbar object.
    """
    fig = ax.get_figure()
    assert fig is not None
    if bbox is None:
        bbox = (1.03, 0, 0.04, 1.0)
    cax = ax.inset_axes(bbox)
    ticks = sm.get_clim() if extreme_ticks_only else None
    cbar = fig.colorbar(sm, cax=cax, label=label, ticks=ticks, **kwargs)
    return cbar


def get_scalar_mappable(
    vmin: float, vmax: float, cmap: str | Colormap
) -> ScalarMappable:
    """Create a ScalarMappable object for a given colormap and range."""
    return ScalarMappable(Normalize(vmin, vmax), cmap)


def get_error_cmap(
    colors: list[str] | None = None, n: int = 256, gamma: float = 1.0
) -> Colormap:
    """Create a colormap for visualization of errors."""
    colors = ["lime", "yellow", "orange", "red"] if colors is None else colors
    return LinearSegmentedColormap.from_list("error_cmap", colors, N=n, gamma=gamma)


def plot_image(
    ax: Axes,
    im: ArrayLike,
    cmap: str = "gray",
    axis_off: bool = True,
    text: str | None = None,
):
    """Plot an image on a given axis."""
    im = im.detach().cpu() if isinstance(im, Tensor) else im
    ax.imshow(im, cmap=cmap)
    if axis_off:
        ax.axis("off")
    if text is not None:
        plot_text(ax, text)


def plot_text(
    ax: Axes,
    text: str,
    pos: tuple[float, float] = (0.01, 0.99),
    ha: str = "left",
    va: str = "top",
    fs: float = 15.0,
    color: str = "w",
    lcolor: str = "k",
    lw: float = 4.0,
    zorder: int = 5,
    **kwargs,
):
    """Plot text on a given axis."""
    pe_ = [pe.withStroke(linewidth=lw, foreground=lcolor)] if lw > 0 else None
    ax.text(
        *pos,
        text,
        ha=ha,
        va=va,
        fontsize=fs,
        color=color,
        zorder=zorder,
        transform=ax.transAxes,
        path_effects=pe_,
        **kwargs,
    )


def plot_heatmap(
    ax: Axes,
    heatmap: np.ndarray | Tensor,
    cmap: str | Colormap = "Spectral_r",
    alpha: float | ArrayLike = 0.2,
    vmin: float | None = None,
    vmax: float | None = None,
    add_colorbar: bool = False,
    contours_every: int = -1,
    text: str | None = None,
    **kwargs,
):
    """Plot heatmaps with optional contours.

    Args:
        ax: Axis to plot the heatmap.
        heatmap: 2D heatmap.
        cmap: Colormap of the Heatmap.
        alpha: Opacity value in [0, 1].
        vmin: Mininmum value to clamp the heatmap.
        vmax: Maximum value to clamp the heatmap.
        contours_every: Interval between contours. If < 1, no contours are plotted.
    """
    cmap = get_error_cmap() if cmap == "error" else cmap
    hm = ax.imshow(
        heatmap,
        cmap=cmap,
        alpha=alpha,
        vmin=vmin,
        vmax=vmax,
    )

    if add_colorbar:
        add_colorbar_to_axes(ax, hm)

    if contours_every > 0:
        plot_contours(
            ax,
            heatmap,
            contours_every=contours_every,
            vmin=vmin,
            vmax=vmax,
            cmap=cmap,
            **kwargs,
        )

    if text is not None:
        plot_text(ax, text)


def plot_contours(
    ax: Axes,
    heatmap: np.ndarray | Tensor,
    contours_every: int = 5,
    vmin: float | None = None,
    vmax: float | None = None,
    cmap: str | Colormap = "Spectral_r",
    contours_colors: ArrayLike | None = None,
    lw: float = 2.0,
    label_units: str = "°",
    label_decimals: int = 0,
    label_fs: float = 16,
    label_lw: float = 1.5,
    label_color: str = "w",
    text: str | None = None,
    **kwargs,
):
    """Plot contours of a heatmap. Adapted and inspired from GeoCalib:
    https://github.com/cvg/GeoCalib/blob/main/geocalib/viz2d.py

    Args:
        ax: Axis to plot the heatmap.
        heatmap: 2D heatmap.
        contours_every: Interval between contours.
        vmin: Mininmum value to clamp the heatmap.
        vmax: Maximum value to clamp the heatmap.
        cmap: Colormap of the Heatmap.
        contours_colors: Color of the contour lines. If not given, it will be infered
            from the colormap and heatmap.
        lw: Width of the contour lines.
        label_units: Units of the heatmap values. Will be used to label the contours.
        label_decimals: Number of decimals to use for the contour labels.
    """
    if contours_every <= 0:
        return
    cmap = get_error_cmap() if cmap == "error" else cmap
    vmin = float(heatmap.min()) if vmin is None else vmin
    vmax = float(heatmap.max()) if vmax is None else vmax
    cs = ax.contour(
        heatmap,
        levels=np.arange(vmin, vmax + contours_every, contours_every),
        colors=contours_colors,
        linewidths=lw,
        vmin=vmin,
        vmax=vmax,
        cmap=cmap if contours_colors is None else None,
    )

    if label_fs > 0:
        clabels_ = ax.clabel(
            cs,
            fmt=f"%.{label_decimals}f{label_units}",
            fontsize=label_fs,
            colors=label_color,
        )
        if label_lw > 0:
            for cl in clabels_:
                cl.set_path_effects([pe.withStroke(linewidth=label_lw, foreground="k")])

    if text is not None:
        plot_text(ax, text)


def plot_2d_arrows(
    ax: Axes,
    directions: np.ndarray | Tensor,
    sample_factor: float = 0.03,
    color_vectors: ColorType | None = "lime",
    color_field: np.ndarray | Tensor | None = None,
    alpha: float | Tensor | np.ndarray = 1,
    cmap: str | Colormap = "Spectral_r",
    add_colorbar: bool = False,
    text: str | None = None,
    **kwargs,
):
    """Plot 2D vector field.

    Args:
        ax: Axis to plot the vector field.
        directions: (H, W, 2) 2D vectors.
        sample_factor: Factor to subsample the vectors. if 1, no subsampling is done.
        color_vectors: Color of the vectors. Ignored if color_field is not None.
        color_field: (H, W) field to color the vectors.
    """
    if not 0 < sample_factor <= 1:
        raise ValueError("sample_factor must be in (0, 1].")
    directions, color_field, alpha = do_detach_cpu(directions, color_field, alpha)

    h, w = directions.shape[:2]
    x, y = np.arange(w), np.arange(h)

    # subsample
    if sample_factor < 1:
        step = int(1 / sample_factor)
        start = step // 2
        x, y = x[start::step], y[start::step]
        directions = directions[start::step, start::step]
        if color_field is not None:
            color_field = color_field[start::step, start::step]
        if isinstance(alpha, (Tensor, np.ndarray)):
            alpha = alpha[start::step, start::step]

    args = (x, y, directions[..., 0], directions[..., 1])
    if color_field is not None:
        args = args + (color_field,)

    sm = ax.quiver(
        *args,
        color=color_vectors,
        cmap=get_error_cmap() if cmap == "error" else cmap,
        alpha=alpha,
        **kwargs,
    )

    if add_colorbar and color_field is not None:
        add_colorbar_to_axes(ax, sm)

    if text is not None:
        plot_text(ax, text)


def plot_radial_vectors(
    ax: Axes,
    bearings: Tensor,
    sample_factor: float = 0.03,
    color_vectors: ColorType = "lime",
    color_field: np.ndarray | Tensor | None = None,
    cmap: str | Colormap = "Spectral_r",
    **kwargs,
):
    """Plot radial vectors.

    Args:
        ax: Axis to plot the radial vectors.
        bearings: (H, W, 3) rays.
        sample_factor: Factor to subsample the vectors. if 1, no subsampling is done.
        color_vectors: Color of the vectors. Ignored if color_field is not None.
        color_field: (H, W) field to color the vectors.
    """
    directions = bearings[..., :2].detach().cpu()
    # check if the y-axis is inverted (e.g. by imshow)
    if ax.yaxis_inverted():
        directions = directions.clone()
        directions[..., 1] = -directions[..., 1]

    if isinstance(color_field, Tensor):
        color_field = color_field.detach().cpu()

    plot_2d_arrows(
        ax,
        directions,
        sample_factor=sample_factor,
        color_vectors=color_vectors,
        color_field=color_field,
        cmap=cmap,
        **kwargs,
    )


def plot_tangent_errors_as_vectors(
    ax: Axes,
    bearings: Tensor,
    bearings_gt: Tensor,
    covs: Tensor | None = None,
    sample_factor: float = 0.03,
    color_vectors: ColorType | None = None,
    cmap: str | Colormap = "error",
    vmin: float | None = 0,
    vmax: float | None = None,
    vmin_log_cov: float = 0,
    **kwargs,
):
    # use tangent space of the ground-truth bearings to compute the errors
    errors = Unit3.logmap(bearings_gt, bearings.detach()).cpu()
    color_field = (
        RAD2DEG * torch.linalg.norm(errors, dim=-1) if color_vectors is None else None
    )
    # check if the y-axis is inverted (e.g. by imshow)
    if ax.yaxis_inverted():
        errors[..., 1] = -errors[..., 1]

    if covs is not None:
        covs = covs.detach().cpu()
        # define opacity based on the (normalized) predicted covariance of each error
        weights = covs_to_confidences(covs, vmin_log=vmin_log_cov, as_log=True)
        alpha = weights.mean(-1)
    else:
        alpha = 1.0

    clim = (None if vmin is None else vmin, None if vmax is None else vmax)
    plot_2d_arrows(
        ax,
        errors,
        sample_factor=sample_factor,
        color_vectors=color_vectors,
        color_field=color_field,
        alpha=alpha,
        cmap=cmap,
        clim=clim,
        **kwargs,
    )


def plot_polar_angles(
    ax: Axes,
    bearings: Tensor,
    contours_every: int = 5,
    vmin: float | None = 0.0,
    vmax: float | None = 90.0,
    cmap: str | Colormap = "Spectral_r",
    only_contours: bool = True,
    **kwargs,
):
    """Plot polar angles as a heatmap.

    Args:
        ax: Axis to plot the heatmap.
        bearings: (H, W, 3) rays.
        contours_every: Interval between contours. If < 1, no contours are plotted.
        vmin: Mininmum value to clamp the heatmap.
        vmax: Maximum value to clamp the heatmap.
        cmap: Colormap of the Heatmap.
        only_contours: If True, only the contours are plotted.
    """
    if only_contours:
        plot_contours(
            ax,
            polar_angles_from_rays(bearings.detach(), as_deg=True).cpu(),
            contours_every=contours_every,
            cmap=cmap,
            vmin=vmin,
            vmax=vmax,
            label_units="°",
            label_decimals=0,
            **kwargs,
        )
    else:
        plot_heatmap(
            ax,
            polar_angles_from_rays(bearings.detach(), as_deg=True).cpu(),
            contours_every=contours_every,
            cmap=cmap,
            vmin=vmin,
            vmax=vmax,
            label_units="°",
            label_decimals=0,
            **kwargs,
        )


def plot_angular_errors(
    ax: Axes,
    bearings: Tensor,
    bearings_gt: Tensor,
    cmap: str | Colormap = "error",
    vmin: float | None = 0.0,
    vmax: float | None = None,
    covs: Tensor | None = None,
    **kwargs,
):
    """Plot geodesic distance errors as heatmaps.

    Args:
        ax: Axis to plot the heatmap.
        bearings: (H, W, 3) rays.
        bearings_gt: (H, W, 3) ground-truth rays.
        cmap: Colormap of the Heatmap.
        vmin: Mininmum value to clamp the error heatmap.
        vmax: Maximum value to clamp the error heatmap.
        covs: (..., N) covariances of the geodesic distances.
    """
    # geodesic (angular) distance between each predicted bearing
    errors = RAD2DEG * Unit3.distance(bearings.detach(), bearings_gt).cpu()  # (..., N)

    if covs is not None:
        covs = covs.detach().cpu()
        # define opacity based on the (normalized) predicted covariance of each error
        weights = covs_to_confidences(
            covs,
            vmin=0.0 if vmin is None else vmin,
            **kwargs,
        )
        alpha = weights.mean(-1)
    else:
        alpha = 0.5

    plot_heatmap(
        ax,
        errors,
        cmap=cmap,
        alpha=alpha,
        vmin=vmin,
        vmax=vmax,
        contours_every=-1,
        **kwargs,
    )


def plot_tangent_errors_as_heatmaps(
    ax_x: Axes,
    ax_y: Axes,
    bearings: Tensor,
    bearings_gt: Tensor,
    cmap: str | Colormap = "error",
    vmin: float | None = 0.0,
    vmax: float | None = None,
    covs: Tensor | None = None,
    **kwargs,
):
    """Plot (absolute) errors defined in the tangent plane of each bearing as heatmaps.

    One heatmap for the x component of the errors and another for the y component.

    Args:
        ax_x: axis for plotting the x component of the errors.
        ax_y: axis for plotting the y component of the errors.
        bearings: (H, W, 3) rays.
        bearings_gt: (H, W, 3) ground-truth rays.
        cmap: Colormap of the Heatmap.
        vmin: In degrees. Mininmum value to clamp the error heatmap.
        vmax: In degrees. Maximum value to clamp the error heatmap.
        covs: (..., N, 2) covariances inthe tangent plane of each bearing.
    """
    # errors in the tangent plane of each ground-truth bearing (H, W, 2)
    errors = RAD2DEG * Unit3.logmap(bearings_gt, bearings.detach()).abs().cpu()

    if covs is not None:
        covs = covs.detach().cpu()
        # define opacity based on the (normalized) predicted covariance of each error
        weights = covs_to_confidences(
            covs,
            vmin=0.0 if vmin is None else vmin,
            **kwargs,
        )
        alpha_x = weights[..., 0]
        alpha_y = weights[..., 1]
    else:
        alpha_x = alpha_y = 0.5

    plot_heatmap(
        ax_x,
        errors[..., 0],
        cmap=cmap,
        alpha=alpha_x,
        vmin=vmin,
        vmax=vmax,
        contours_every=-1,
        **kwargs,
    )

    plot_heatmap(
        ax_y,
        errors[..., 1],
        cmap=cmap,
        alpha=alpha_y,
        vmin=vmin,
        vmax=vmax,
        contours_every=-1,
        **kwargs,
    )


def covs_to_confidences(
    covs: Tensor,
    as_log: bool = True,
    vmin: float | None = 0,
    vmin_log: float | None = 0,
    **kwargs,
):
    """Convert covariances to confidences in [0, 1] for visualization.

    The intuition for selecting the minimum value in the log case (the default) is to
    use vmin_log as the upper-bound angular uncertainty (in degrees) that ideally and
    approximately corresponds to correct predictions. For example, when vmin_log is set
    to 0, uncertainties below 1 degree (1e0) will have full opacity (1.0), while each
    subsequent order of magnitude increase in uncertainty will reduce the opacity
    linearly until the maximum value is reached.
    """
    eps = torch.finfo(covs.dtype).eps
    stds = covs.clamp(eps).sqrt() * RAD2DEG
    if as_log:
        vmin_log = 0 if vmin_log is None else vmin_log
        weights = torch.log10(stds).clamp(vmin_log)
        weights = 1 - (weights - vmin_log) / (weights.max() - vmin_log)
    else:
        vmin = 0 if vmin is None else vmin
        weights = stds.clamp(vmin)
        weights = 1 - (weights - vmin) / (weights.max() - vmin)
    return weights


def plot_uncertainties_as_heatmap(
    ax: Axes | tuple[Axes, Axes],
    covs: Tensor,
    aggregator: str | Callable | None = "mean",
    cmap: str | Colormap = "turbo_r",
    vmin_log: float = 0,
    alpha: float = 0.5,
    log_normalize: bool = True,
    **kwargs,
):
    """Plot uncertainties as heatmaps.

    Args:
        ax: Axis or tuple of axes to plot the uncertainties.
        covs: (H, W, 2) covariances in the tangent plane of each bearing.
        aggregator: Aggregator function to use when plotting the uncertainties. Default
            is "mean".
        cmap: Colormap of the Heatmap.
        vmin_log: Minimum value to clamp the log uncertainties.
        alpha: Opacity value in [0, 1].
    """
    if aggregator is None and isinstance(ax, Axes):
        raise ValueError("aggregator must be given when a single axis is provided.")

    eps = torch.finfo(covs.dtype).eps
    # we visualize the confidences as the inverse of the standard deviations
    std = RAD2DEG * covs.detach().clamp(eps).sqrt()
    if log_normalize:
        std = torch.log10(std).clamp(vmin_log)
        # std = (std - std.min()) / (std.max() - std.min())  # bound values
    std = std.cpu()

    if isinstance(ax, Axes):
        # when just one axis is given, plot the aggregation of the uncertainties
        if isinstance(aggregator, str):
            std = operator.attrgetter(aggregator)(torch)(std, dim=-1)
        elif callable(aggregator):
            std = aggregator(std)
        else:
            raise ValueError("aggregator must be a function or a string.")
        plot_heatmap(ax, std, cmap=cmap, alpha=alpha, **kwargs)
        return

    # when two axes are given, plot the x and y components of the uncertainties
    plot_heatmap(ax[0], std[..., 0], cmap=cmap, alpha=alpha, **kwargs)
    plot_heatmap(ax[1], std[..., 1], cmap=cmap, alpha=alpha, **kwargs)


def plot_uncertainties_as_ellipses():
    raise NotImplementedError