fMRI-foundation-model/src/mae_utils/visualize.py

import io
from typing import Any, Optional

import torch
from matplotlib import pyplot as plt
from matplotlib.figure import Figure
from PIL import Image

plt.rcParams["figure.dpi"] = 150


def plot_mask_pred(
    model: torch.nn.Module,
    imgs: torch.Tensor,
    pred: torch.Tensor,
    mask: torch.Tensor,
    mean: Optional[Any] = None,
    std: Optional[Any] = None,
    nrow: int = 8,
):
    # imgs: [N, C, T, H, W]
    # pred: [N, t*h*w, u*p*p*C]
    # mask: [N, t*h*w], 0 is keep, 1 is remove,
    target = torch.index_select(
        imgs,
        2,
        torch.linspace(
            0,
            imgs.shape[2] - 1,
            model.pred_t_dim,
        )
        .long()
        .to(imgs.device),
    )
    target = torch.einsum("ncthw->nthwc", target)
    target = target.flatten(0, 1)[:nrow].cpu()
    
    mask = mask.unsqueeze(-1).repeat(
        1, 1, pred.shape[-1]
    )  # (N, T*H*W, p*p*c)
    mask = model.unpatchify(mask)  # 1 is removing, 0 is keeping
    mask = torch.einsum("ncthw->nthwc", mask).cpu()
    mask = mask.flatten(0, 1)[:nrow].cpu()

    pred = pred.detach()
    pred = model.unpatchify(pred)
    pred = torch.einsum("ncthw->nthwc", pred).cpu()
    pred = pred.flatten(0, 1)[:nrow].cpu()

    # masked image
    im_masked = target * (1 - mask)

    # MAE reconstruction pasted with visible patches
    im_paste = target * (1 - mask) + pred * mask

    if model.img_mask is not None:
        img_mask = model.img_mask.cpu()
    else:
        img_mask = None

    H, W = target.shape[1:3]
    ploth = 2.0
    plotw = (W / H) * ploth
    nrow = len(target)
    ncol = 3
    fig, axs = plt.subplots(
        nrow, ncol, figsize=(plotw * ncol, ploth * nrow), squeeze=False
    )

    for ii in range(nrow):
        plt.sca(axs[ii, 0])
        imshow(im_masked[ii], mean=mean, std=std, mask=img_mask)

        plt.sca(axs[ii, 1])
        imshow(im_paste[ii], mean=mean, std=std, mask=img_mask)

        plt.sca(axs[ii, 2])
        imshow(target[ii], mean=mean, std=std, mask=img_mask)

    plt.tight_layout(pad=0.25)
    return fig


def imshow(
    image: torch.Tensor,
    mean: Optional[Any] = None,
    std: Optional[Any] = None,
    mask: Optional[torch.Tensor] = None,
    **kwargs,
):
    # image: (H, W, C)
    assert image.shape[2] in (1, 3)
    if image.shape[2] == 1:
        kwargs = {
            "cmap": "gray",
            "vmin": 0.0,
            "vmax": 1.0,
            "interpolation": "nearest",
            **kwargs,
        }
    if mean is not None:
        mean = torch.as_tensor(mean)
        std = torch.as_tensor(std)
        image = torch.clip(image * std + mean, 0.0, 1.0)
    if mask is not None:
        image = mask.unsqueeze(-1) * image
    plt.imshow(image, **kwargs)
    plt.axis("off")


def fig2pil(fig: Figure, format: str = "png") -> Image.Image:
    with io.BytesIO() as f:
        fig.savefig(f, format=format)
        f.seek(0)
        img = Image.open(f)
        img.load()
    return img