Spaces:
Running on Zero
Running on Zero
File size: 2,281 Bytes
e793773 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 | """Pipeline unit that converts camera trajectories into SCoPE coordinates."""
from __future__ import annotations
from typing import Any
import torch
from einops import repeat
from diffsynth.utils import PipelineUnit
from scope.geometry import compute_plucker_coordinates
class SCoPECameraUnit(PipelineUnit):
"""Build token-aligned sightline coordinates from a pinhole camera path."""
def __init__(self) -> None:
super().__init__(input_params=("height", "width", "camera_control_panshot"))
def process(
self,
pipe: Any,
height: int,
width: int,
camera_control_panshot: dict[str, Any] | None,
) -> dict[str, dict[str, torch.Tensor | int]]:
if camera_control_panshot is None:
return {}
if getattr(pipe.dit, "camera_condition", "none") != "scope":
return {}
pose = camera_control_panshot["pose"][:, ::4] # [B, T_latent, 3, 4]
x_fov = camera_control_panshot["x_fov"]
if not torch.is_tensor(x_fov):
x_fov = torch.tensor([float(x_fov)], device=pose.device, dtype=pose.dtype)
x_fov = x_fov.to(device=pose.device, dtype=pose.dtype).reshape(-1)
c2w = torch.eye(4, device=pose.device, dtype=pose.dtype)
c2w = repeat(c2w, "i j -> b t i j", b=pose.shape[0], t=pose.shape[1]).clone()
c2w[..., :3, :4] = pose
patch_factor = pipe.vae.upsampling_factor * 2
patches_x = width // patch_factor
patches_y = height // patch_factor
focal = (width * 0.5) / torch.tan(x_fov * 0.5)
intrinsics = torch.zeros(
(pose.shape[0], pose.shape[1], 3, 3), device=pose.device, dtype=pose.dtype
)
intrinsics[..., 0, 0] = focal[:, None]
intrinsics[..., 1, 1] = focal[:, None]
intrinsics[..., 0, 2] = width * 0.5
intrinsics[..., 1, 2] = height * 0.5
intrinsics[..., 2, 2] = 1.0
coordinates = compute_plucker_coordinates(
c2w,
intrinsics,
patches_y,
patches_x,
height,
width,
)
return {
"control_camera_dit_input": {
"plucker_6d": coordinates,
"num_frames": pose.shape[1],
}
}
|