diffsynth/models/camera.py

from functools import partial

import torch

from .prope import _rope_precompute_coeffs


def intrinsics_to_K(intrinsics):
    batch_size, num_frames, _ = intrinsics.shape
    K = torch.zeros((batch_size, num_frames, 3, 3), dtype=intrinsics.dtype, device=intrinsics.device)
    K[..., 0, 0] = intrinsics[..., 0]
    K[..., 1, 1] = intrinsics[..., 1]
    K[..., 0, 2] = intrinsics[..., 2]
    K[..., 1, 2] = intrinsics[..., 3]
    K[..., 2, 2] = 1.0
    return K


def get_recammaster_embedding(cam_c2w, height, width):
    batch_size, num_frames, _, _ = cam_c2w.shape
    cam_emb = cam_c2w[:, :, :3, :].reshape(batch_size, num_frames, 1, 1, 12)  # b f 4 4 -> b f 3 4 -> b f 1 1 12
    cam_emb = cam_emb.repeat(1, 1, height, width, 1)  # b f 1 1 12 -> b f h w 12
    return cam_emb


def get_plucker_embedding(intrinsics, cam_c2w, height, width, height_dit=None, width_dit=None, flip_flag=None):
    """
    Computes the Plucker embedding given camera intrinsics and extrinsics

    Params:
        intrinsics (torch.Tensor): Camera intrinsics, shape b f 4 -> b f [ fx fy cx cy ]
        cam_c2w (torch.Tensor): Camera extrinsics, shape b f 4 4
        ...

    Returns:
        plucker (torch.Tensor): Plucker embedding, shape b f h w 6

    From AC3D: https://github.com/snap-research/ac3d/blob/3c1e29e688f4a6d0f0ad41f1bf75d2eab709dac2/training/controlnet_datasets_camera.py#L111
    """

    custom_meshgrid = partial(torch.meshgrid, indexing="ij")

    batch_size, num_frames = intrinsics.shape[:2]

    use_dit_hw = True
    if height_dit is None or width_dit is None:
        use_dit_hw = False
        height_dit = height
        width_dit = width
    else:
        patch_height = height / height_dit
        patch_width = width / width_dit

    j, i = custom_meshgrid(
        torch.linspace(0, height_dit - 1, height_dit, device=cam_c2w.device, dtype=cam_c2w.dtype),
        torch.linspace(0, width_dit - 1, width_dit, device=cam_c2w.device, dtype=cam_c2w.dtype),
    )
    # b f (h w)
    i = i.reshape([1, 1, height_dit * width_dit]).expand([batch_size, num_frames, height_dit * width_dit]) + 0.5
    j = j.reshape([1, 1, height_dit * width_dit]).expand([batch_size, num_frames, height_dit * width_dit]) + 0.5

    if use_dit_hw:
        i = i * patch_width + (patch_width / 2)
        j = j * patch_height + (patch_height / 2)

    n_flip = torch.sum(flip_flag).item() if flip_flag is not None else 0
    if n_flip > 0:
        j_flip, i_flip = custom_meshgrid(
            torch.linspace(0, height_dit - 1, height_dit, device=cam_c2w.device, dtype=cam_c2w.dtype),
            torch.linspace(width_dit - 1, 0, width_dit, device=cam_c2w.device, dtype=cam_c2w.dtype)
        )
        i_flip = i_flip.reshape([1, 1, height_dit * width_dit]).expand(batch_size, 1, height_dit * width_dit) + 0.5
        j_flip = j_flip.reshape([1, 1, height_dit * width_dit]).expand(batch_size, 1, height_dit * width_dit) + 0.5
        if use_dit_hw:
            i_flip = i_flip * patch_width + (patch_width / 2)
            j_flip = j_flip * patch_height + (patch_height / 2)

        i[:, flip_flag, ...] = i_flip
        j[:, flip_flag, ...] = j_flip

    fx, fy, cx, cy = intrinsics.chunk(4, dim=-1)  # b f 1

    zs = torch.ones_like(i)  # b f (h w)
    xs = (i - cx) / fx * zs
    ys = (j - cy) / fy * zs
    zs = zs.expand_as(ys)

    directions = torch.stack((xs, ys, zs), dim=-1)  # b f (h w) 3
    directions = directions / directions.norm(dim=-1, keepdim=True)  # b f (h w) 3

    rays_d = directions @ cam_c2w[..., :3, :3].transpose(-1, -2)  # b f (h w) 3
    rays_o = cam_c2w[..., :3, 3]  # b f 3
    rays_o = rays_o[:, :, None].expand_as(rays_d)  # b f (h w) 3
    # cam_c2w @ directions
    rays_dxo = torch.cross(rays_o, rays_d, dim=-1)  # b f (h w) 3
    plucker = torch.cat([rays_dxo, rays_d], dim=-1)
    plucker = plucker.reshape(batch_size, cam_c2w.shape[1], height_dit, width_dit, 6)  # b f h w 6
    return plucker


def get_prope_dict(
    cam_c2w, intrinsics, height, width, height_dit, width_dit, time_division_factor=4,
    precompute_coeffs=False, coeffs_x=None, coeffs_y=None, head_dim=None, num_frames_multiplier=2,
):
    cam_c2w = cam_c2w[:, ::time_division_factor]
    K = intrinsics_to_K(intrinsics)[:, ::time_division_factor]

    dtype = cam_c2w.dtype
    device = cam_c2w.device

    batch_size, num_frames, _, _ = cam_c2w.shape
    num_frames = num_frames * num_frames_multiplier  # For ReCamMaster-type training
    if precompute_coeffs:
        if coeffs_x is None:
            assert head_dim is not None
            coeffs_x = _rope_precompute_coeffs(
                torch.tile(torch.arange(width_dit, dtype=dtype, device=device), (height_dit * num_frames,)),
                freq_base=100.0,
                freq_scale=1.0,
                feat_dim=head_dim // 4,
            )
        if coeffs_y is None:
            assert head_dim is not None
            coeffs_y = _rope_precompute_coeffs(
                torch.tile(
                    torch.repeat_interleave(
                        torch.arange(height_dit, dtype=dtype, device=device), width_dit
                    ),
                    (num_frames,),
                ),
                freq_base=100.0,
                freq_scale=1.0,
                feat_dim=head_dim // 4,
            )

    prope_dict = {
        "viewmats": cam_c2w,
        "Ks": K,
        "patches_x": width_dit,
        "patches_y": height_dit,
        "image_width": width,
        "image_height": height,
        "coeffs_x": coeffs_x,
        "coeffs_y": coeffs_y,
    }
    return prope_dict