scope/camera.py

"""Pipeline unit that converts camera trajectories into SCoPE coordinates."""

from __future__ import annotations

from typing import Any

import torch
from einops import repeat

from diffsynth.utils import PipelineUnit
from scope.geometry import compute_plucker_coordinates


class SCoPECameraUnit(PipelineUnit):
    """Build token-aligned sightline coordinates from a pinhole camera path."""

    def __init__(self) -> None:
        super().__init__(input_params=("height", "width", "camera_control_panshot"))

    def process(
        self,
        pipe: Any,
        height: int,
        width: int,
        camera_control_panshot: dict[str, Any] | None,
    ) -> dict[str, dict[str, torch.Tensor | int]]:
        if camera_control_panshot is None:
            return {}
        if getattr(pipe.dit, "camera_condition", "none") != "scope":
            return {}

        pose = camera_control_panshot["pose"][:, ::4]  # [B, T_latent, 3, 4]
        x_fov = camera_control_panshot["x_fov"]
        if not torch.is_tensor(x_fov):
            x_fov = torch.tensor([float(x_fov)], device=pose.device, dtype=pose.dtype)
        x_fov = x_fov.to(device=pose.device, dtype=pose.dtype).reshape(-1)

        c2w = torch.eye(4, device=pose.device, dtype=pose.dtype)
        c2w = repeat(c2w, "i j -> b t i j", b=pose.shape[0], t=pose.shape[1]).clone()
        c2w[..., :3, :4] = pose

        patch_factor = pipe.vae.upsampling_factor * 2
        patches_x = width // patch_factor
        patches_y = height // patch_factor
        focal = (width * 0.5) / torch.tan(x_fov * 0.5)
        intrinsics = torch.zeros(
            (pose.shape[0], pose.shape[1], 3, 3), device=pose.device, dtype=pose.dtype
        )
        intrinsics[..., 0, 0] = focal[:, None]
        intrinsics[..., 1, 1] = focal[:, None]
        intrinsics[..., 0, 2] = width * 0.5
        intrinsics[..., 1, 2] = height * 0.5
        intrinsics[..., 2, 2] = 1.0

        coordinates = compute_plucker_coordinates(
            c2w,
            intrinsics,
            patches_y,
            patches_x,
            height,
            width,
        )
        return {
            "control_camera_dit_input": {
                "plucker_6d": coordinates,
                "num_frames": pose.shape[1],
            }
        }