from math import pi, sqrt import matplotlib.pyplot as plt import torch import torch.nn.functional as F import anycalib.visualization.viz_2d as viz_2d from anycalib.cameras.base import BaseCamera from anycalib.manifolds import Unit3 RAD2DEG = 180 / pi def create_figures(im_size, nrows, ncols, min_fig_size: float = 4.0): h, w = im_size plot_w = min_fig_size if w < h else min_fig_size * w / h plot_h = min_fig_size if h < w else min_fig_size * h / w fig, axs = plt.subplots( nrows, ncols, figsize=(plot_w * ncols, plot_h * nrows), squeeze=False, ) return fig, axs def make_radial_figure( pred: dict, data: dict, n_pairs: int = 2, min_fig_size: float = 4.0, show_radial_vecs: bool = False, ) -> dict: """Image grid of radial vectors plots.""" titles = ("Image", "Polar angles GT", "Pred") if show_radial_vecs: titles = titles + ("Radial vecs GT", "Pred") if "tangent_coords" in pred: titles = titles + ("Tangent θx GT", "Pred", "Tangent θy GT", "Pred") # image grid n_pairs = min(n_pairs, len(data["image"])) b, _, h, w = data["image"].shape fig, axs = create_figures((h, w), n_pairs, len(titles), min_fig_size=min_fig_size) images = data["image"].permute(0, 2, 3, 1).clamp(0, 1) rays_gt = data["rays"].view(b, h, w, 3) rays = pred["rays"].view(b, h, w, 3) if "tangent_coords" in pred: tcoords_gt = (RAD2DEG * Unit3.logmap_at_z1(rays_gt)).cpu() tcoords = (RAD2DEG * pred["tangent_coords"].detach().view(b, h, w, 2)).cpu() tc_kwargs = { "contours_every": 10, "cmap": "Spectral_r", "vmin": -90, "vmax": 90, } for i in range(n_pairs): axr = axs[i] for j, title in enumerate(titles): viz_2d.plot_image(axr[j], images[i]) if i == 0: axr[j].set_title(title) # polar angles viz_2d.plot_polar_angles(axr[1], rays_gt[i]) viz_2d.plot_polar_angles(axr[2], rays[i]) j = 3 if show_radial_vecs: # quiver plots with the same scale as the ground-truth viz_2d.plot_radial_vectors(axr[j], rays_gt[i], color_vectors="lime") fig.canvas.draw() # ensure the scale attribute of the quiver plot is present viz_2d.plot_radial_vectors( axr[j + 1], rays[i], color_vectors="orange", scale=axr[1].collections[-1].scale, ) j = 5 if "tangent_coords" in pred: tc_gt = tcoords_gt[i] tc = tcoords[i] viz_2d.plot_contours(axr[j], tc_gt[..., 0], **tc_kwargs) viz_2d.plot_contours(axr[j + 1], tc[..., 0], **tc_kwargs) viz_2d.plot_contours(axr[j + 2], tc_gt[..., 1], **tc_kwargs) viz_2d.plot_contours(axr[j + 3], tc[..., 1], **tc_kwargs) fig.tight_layout() return {"radial": fig} def make_errors_figure( pred: dict, data: dict, n_pairs: int = 2, min_fig_size: float = 4.0, log_normalize: bool = False, agg_fn: str | None = None, # "prod", ) -> dict: """Image grid of error plots.""" titles = ("Image", "Tangent Error", "Angular Error") suptitle = "Errors" if "log_covs" in pred: suptitle = "Errors and Uncertainties" unc_str = "log-uncertainty" if log_normalize else "uncertainty" titles += (f"{unc_str}-x", f"{unc_str}-y") if agg_fn is None else (unc_str,) if "weights" in pred: titles += ("alpha",) # image grid n_pairs = min(n_pairs, len(data["image"])) b, _, h, w = data["image"].shape fig, axs = create_figures((h, w), n_pairs, len(titles), min_fig_size=min_fig_size) fig.suptitle(suptitle) images = data["image"].permute(0, 2, 3, 1).clamp(0, 1) rays_gt = data["rays"].view(b, h, w, 3) rays = pred["rays"].view(b, h, w, 3) log_covs = pred.get("log_covs", None) if log_covs is not None: covs = log_covs.exp().view(b, h, w, 2) if "weights" in pred: # compute "alpha(s)" of mixture model alpha = torch.softmax(pred["weights"], dim=-1)[..., 0] # (b, 1 or h*w) alpha = ( alpha.expand(-1, h * w).view(b, h, w).cpu() if alpha.shape[-1] == 1 else alpha.view(b, h, w).cpu() ) for i in range(n_pairs): axr = axs[i] for j, title in enumerate(titles): viz_2d.plot_image(axr[j], images[i]) if i == 0: axr[j].set_title(title) # errors viz_2d.plot_tangent_errors_as_vectors(axr[1], rays[i], rays_gt[i]) viz_2d.plot_angular_errors(axr[2], rays[i], rays_gt[i], add_colorbar=True) # uncertainties if log_covs is not None: axr_u = axr[3] if isinstance(agg_fn, str) else (axr[3], axr[4]) viz_2d.plot_uncertainties_as_heatmap( axr_u, covs[i], log_normalize=log_normalize, aggregator=agg_fn, add_colorbar=True, ) if "weights" in pred: viz_2d.plot_heatmap(axr[-1], alpha[i], add_colorbar=True) fig.tight_layout() return {"errors": fig} def make_editmaps_figure( pred: dict, data: dict, n_pairs: int = 2, min_fig_size: float = 4.0 ): """Image grid of editmap plots.""" titles = ("Image", "Pix AR Error", "Uncertainty", "Radii GT [pix]", "Pred") n_pairs = min(n_pairs, len(data["image"])) _, _, h, w = data["image"].shape rmax = 0.5 * sqrt(h**2 + w**2) fig, axs = create_figures((h, w), n_pairs, len(titles), min_fig_size=min_fig_size) images = data["image"].permute(0, 2, 3, 1).clamp(0, 1) _, _, hp, wp = pred["pix_ar_map"].shape assert h / hp == w / wp and h / hp >= 1 pix_ar_gt = data["pix_ar"] # (b,) radii = (h / hp) * pred["radii"].detach() # (b, hp, wp) # upsample to image resolution radii = F.interpolate(radii[:, None], (h, w), mode="bilinear", align_corners=False)[:, 0].cpu() # fmt:skip pix_ar = F.interpolate( pred["pix_ar_map"].detach(), (h, w), mode="bilinear", align_corners=False ) # (b, 2, h, w) pix_ar_err = (pix_ar[:, 0] - pix_ar_gt[:, None, None]).abs().cpu() # (b, h, w) pix_ar_unc = torch.exp(pix_ar[:, 1]).cpu() radii_gt = torch.linalg.norm( BaseCamera.pixel_grid_coords(h, w, data["cxcy_gt"], 0.5) - data["cxcy_gt"][:, None, None], dim=-1, ).cpu() for i in range(n_pairs): axr = axs[i] for j, title in enumerate(titles): viz_2d.plot_image(axr[j], images[i]) if i == 0: axr[j].set_title(title) viz_2d.plot_heatmap(axr[1], pix_ar_err[i], alpha=0.5, add_colorbar=True, cmap="error") # fmt: skip viz_2d.plot_text(axr[1], f"GT: {pix_ar_gt[i].item():.1f}") viz_2d.plot_heatmap(axr[2], pix_ar_unc[i], alpha=0.5, add_colorbar=True, cmap="turbo_r") # fmt: skip viz_2d.plot_contours(axr[3], radii_gt[i], vmin=0, vmax=rmax, contours_every=25, label_units="") # fmt: skip viz_2d.plot_contours(axr[4], radii[i], vmin=0, vmax=rmax, contours_every=25, label_units="") # fmt: skip fig.tight_layout() return {"editmaps": fig} def make_batch_figures( pred: dict, data: dict, n_pairs: int = 3, min_fig_size: float = 3.0 ) -> dict: """Create figures for debugging""" figs = make_radial_figure(pred, data, n_pairs, min_fig_size=min_fig_size) figs |= make_errors_figure(pred, data, n_pairs, min_fig_size=min_fig_size) if "pix_ar_map" in pred and "radii" in pred: figs |= make_editmaps_figure(pred, data, n_pairs, min_fig_size=min_fig_size) return figs