dp2/generator/utils.py

import torch
import tops
import torch
from torch.cuda.amp import custom_bwd, custom_fwd


@torch.no_grad()
def spatial_embed_keypoints(keypoints: torch.Tensor, x):
    tops.assert_shape(keypoints, (None, None, 3))
    B, N_K, _ = keypoints.shape
    H, W = x.shape[-2:]
    keypoint_spatial = torch.zeros(keypoints.shape[0], N_K, H, W, device=keypoints.device, dtype=torch.float32)
    x, y, visible = keypoints.chunk(3, dim=2)
    x = (x * W).round().long().clamp(0, W-1)
    y = (y * H).round().long().clamp(0, H-1)
    kp_idx = torch.arange(0, N_K, 1, device=keypoints.device, dtype=torch.long).view(1, -1, 1).repeat(B, 1, 1)
    pos = (kp_idx*(H*W) + y*W + x + 1)
    # Offset all by 1 to index invisible keypoints as 0
    pos = (pos * visible.round().long()).squeeze(dim=-1)
    keypoint_spatial = torch.zeros(keypoints.shape[0], N_K*H*W+1, device=keypoints.device, dtype=torch.float32)
    keypoint_spatial.scatter_(1, pos, 1)
    keypoint_spatial = keypoint_spatial[:, 1:].view(-1, N_K, H, W)
    return keypoint_spatial


class MaskOutput(torch.autograd.Function):

    @staticmethod
    @custom_fwd
    def forward(ctx, x_real, x_fake, mask):
        ctx.save_for_backward(mask)
        out = x_real * mask + (1-mask) * x_fake
        return out

    @staticmethod
    @custom_bwd
    def backward(ctx, grad_output):
        fake_grad = grad_output
        mask, = ctx.saved_tensors
        fake_grad = fake_grad * (1 - mask)
        known_percentage = mask.view(mask.shape[0], -1).mean(dim=1)
        fake_grad = fake_grad / (1-known_percentage).view(-1, 1, 1, 1)
        return None, fake_grad, None


def mask_output(scale_grad, x_real, x_fake, mask):
    if scale_grad:
        return MaskOutput.apply(x_real, x_fake, mask)
    return x_real * mask + (1-mask) * x_fake