LoomNet / loomnet /utils /project.py
Giulio Federico
Upload of the code without weights
2039c18
Raw History Blame Contribute Delete
18.2 kB
from einops import rearrange, repeat
import torch
import torch.nn.functional as F
import copy
def compute_rel_transform(c2w, ray_origins, ray_directions):
"""
c2w: (b x f) x 4 x 4
ray_origins: (b x f) x (h x w) x 3
ray_directions: (b x f) x (h x w) x 3
return:
canonical_w2c: (b x f) x (h x w) x 3 x 4
"""
upv = c2w[:, :3, 1] # (b x f) x 3
upv = rearrange(upv, "bf i -> bf 1 i") # (b x f) x 1 x 3
rdotup = (ray_directions * upv).sum(-1, keepdims=True) # (b x f) x (h x w) x 1
orthoup = upv - rdotup * ray_directions # 史密斯正交化
orthoup = F.normalize(orthoup, dim=-1) # (b x f) x (h x w) x 3
vec0 = torch.cross(orthoup, ray_directions) # (b x f) x (h x w) x 3
vec0 = F.normalize(vec0, dim=-1) # (b x f) x (h x w) x 3
r_relatives = torch.stack(
(vec0, orthoup, ray_directions), dim=-1
) # (b x f) x (h x w) x 3 x 3
r_relatives_T = r_relatives.transpose(-1, -2) # (b x f) x (h x w) x 3 x 3
translation_relative = torch.einsum(
"... i j, ... j -> ... i", -r_relatives_T, ray_origins
).unsqueeze(
-1
) # (b x f) x (h x w) x 3
canonical_w2c = torch.cat(
(r_relatives_T, translation_relative), dim=-1
) # (b x f) x (h x w) x 3 x 4
return canonical_w2c
def ray_sample(cam2world_matrix, intrinsics, resolution):
"""
Create batches of rays and return origins and directions.
cam2world_matrix: (N, 4, 4)
intrinsics: (N, 3, 3)
resolution: int
ray_origins: (N, M, 3)
ray_dirs: (N, M, 3)
"""
N, M = cam2world_matrix.shape[0], resolution**2
cam_locs_world = cam2world_matrix[:, :3, 3]
fx = intrinsics[:, 0, 0]
fy = intrinsics[:, 1, 1]
cx = intrinsics[:, 0, 2]
cy = intrinsics[:, 1, 2]
sk = intrinsics[:, 0, 1]
uv = torch.stack(
torch.meshgrid(
torch.arange(
resolution, dtype=torch.float32, device=cam2world_matrix.device
),
torch.arange(
resolution, dtype=torch.float32, device=cam2world_matrix.device
),
indexing="ij",
)
) * (1.0 / resolution) + (0.5 / resolution)
uv = repeat(uv, "c h w -> b (h w) c", b=N)
x_cam = uv[:, :, 0].view(N, -1)
y_cam = uv[:, :, 1].view(N, -1)
z_cam = torch.ones((N, M), device=cam2world_matrix.device)
x_lift = (
(
x_cam
- cx.unsqueeze(-1)
+ cy.unsqueeze(-1) * sk.unsqueeze(-1) / fy.unsqueeze(-1)
- sk.unsqueeze(-1) * y_cam / fy.unsqueeze(-1)
)
/ fx.unsqueeze(-1)
* z_cam
)
y_lift = (y_cam - cy.unsqueeze(-1)) / fy.unsqueeze(-1) * z_cam
cam_rel_points = torch.stack(
(x_lift, y_lift, z_cam, torch.ones_like(z_cam)), dim=-1
)
world_rel_points = torch.bmm(
cam2world_matrix, cam_rel_points.permute(0, 2, 1)
).permute(0, 2, 1)[:, :, :3]
ray_dirs = world_rel_points - cam_locs_world[:, None, :]
ray_dirs = torch.nn.functional.normalize(ray_dirs, dim=2)
ray_origins = cam_locs_world.unsqueeze(1).repeat(1, ray_dirs.shape[1], 1)
return ray_origins, ray_dirs
def linspace(start: torch.Tensor, stop: torch.Tensor, num: int):
"""
Creates a tensor of shape [num, *start.shape] whose values are evenly spaced from start to end, inclusive.
Replicates but the multi-dimensional bahaviour of numpy.linspace in PyTorch.
"""
# create a tensor of 'num' steps from 0 to 1
steps = torch.arange(num, dtype=torch.float32, device=start.device) / (num - 1)
# reshape the 'steps' tensor to [-1, *([1]*start.ndim)] to allow for broadcastings
# - using 'steps.reshape([-1, *([1]*start.ndim)])' would be nice here but torchscript
# "cannot statically infer the expected size of a list in this contex", hence the code below
for i in range(start.ndim):
steps = steps.unsqueeze(-1)
# the output starts at 'start' and increments until 'stop' in each dimension
out = start[None] + steps * (stop - start)[None]
return out
def sample_stratified(
ray_origins,
ray_start,
ray_end,
depth_resolution,
disparity_space_sampling=False,
return_depth=False,
):
"""
Return depths of approximately uniformly spaced samples along rays.
"""
N, M, _ = ray_origins.shape
if disparity_space_sampling:
depths_coarse = (
torch.linspace(0, 1, depth_resolution, device=ray_origins.device)
.reshape(1, 1, depth_resolution, 1)
.repeat(N, M, 1, 1)
)
if not return_depth:
depth_delta = 1 / (depth_resolution - 1)
depths_coarse += torch.rand_like(depths_coarse) * depth_delta
depths_coarse = 1.0 / (
1.0 / ray_start * (1.0 - depths_coarse) + 1.0 / ray_end * depths_coarse
)
else:
if type(ray_start) == torch.Tensor:
depths_coarse = linspace(ray_start, ray_end, depth_resolution).permute(
1, 2, 0, 3
)
if not return_depth:
depth_delta = (ray_end - ray_start) / (depth_resolution - 1)
depths_coarse += torch.rand_like(depths_coarse) * depth_delta[..., None]
else:
depths_coarse = (
torch.linspace(
ray_start, ray_end, depth_resolution, device=ray_origins.device
)
.reshape(1, 1, depth_resolution, 1)
.repeat(N, M, 1, 1)
)
if not return_depth:
depth_delta = (ray_end - ray_start) / (depth_resolution - 1)
depths_coarse += torch.rand_like(depths_coarse) * depth_delta
return depths_coarse
def get_ray_limits_box(rays_o: torch.Tensor, rays_d: torch.Tensor, box_side_length):
"""
Author: Petr Kellnhofer
Intersects rays with the [-1, 1] NDC volume.
Returns min and max distance of entry.
Returns -1 for no intersection.
https://www.scratchapixel.com/lessons/3d-basic-rendering/minimal-ray-tracer-rendering-simple-shapes/ray-box-intersection
"""
o_shape = rays_o.shape
rays_o = rays_o.detach().reshape(-1, 3)
rays_d = rays_d.detach().reshape(-1, 3)
bb_min = [
-1 * (box_side_length / 2),
-1 * (box_side_length / 2),
-1 * (box_side_length / 2),
]
bb_max = [
1 * (box_side_length / 2),
1 * (box_side_length / 2),
1 * (box_side_length / 2),
]
bounds = torch.tensor([bb_min, bb_max], dtype=rays_o.dtype, device=rays_o.device)
is_valid = torch.ones(rays_o.shape[:-1], dtype=bool, device=rays_o.device)
# Precompute inverse for stability.
invdir = 1 / rays_d
sign = (invdir < 0).long()
# Intersect with YZ plane.
tmin = (bounds.index_select(0, sign[..., 0])[..., 0] - rays_o[..., 0]) * invdir[
..., 0
]
tmax = (bounds.index_select(0, 1 - sign[..., 0])[..., 0] - rays_o[..., 0]) * invdir[
..., 0
]
# Intersect with XZ plane.
tymin = (bounds.index_select(0, sign[..., 1])[..., 1] - rays_o[..., 1]) * invdir[
..., 1
]
tymax = (
bounds.index_select(0, 1 - sign[..., 1])[..., 1] - rays_o[..., 1]
) * invdir[..., 1]
# Resolve parallel rays.
is_valid[torch.logical_or(tmin > tymax, tymin > tmax)] = False
# Use the shortest intersection.
tmin = torch.max(tmin, tymin)
tmax = torch.min(tmax, tymax)
# Intersect with XY plane.
tzmin = (bounds.index_select(0, sign[..., 2])[..., 2] - rays_o[..., 2]) * invdir[
..., 2
]
tzmax = (
bounds.index_select(0, 1 - sign[..., 2])[..., 2] - rays_o[..., 2]
) * invdir[..., 2]
# Resolve parallel rays.
is_valid[torch.logical_or(tmin > tzmax, tzmin > tmax)] = False
# Use the shortest intersection.
tmin = torch.max(tmin, tzmin)
tmax = torch.min(tmax, tzmax)
# Mark invalid.
tmin[torch.logical_not(is_valid)] = -1
tmax[torch.logical_not(is_valid)] = -2
return tmin.reshape(*o_shape[:-1], 1), tmax.reshape(*o_shape[:-1], 1)
def sample_points(ray_origins, ray_directions, rendering_options, return_depth=False):
if rendering_options["ray_start"] == rendering_options["ray_end"] == "auto":
ray_start, ray_end = get_ray_limits_box(
ray_origins, ray_directions, box_side_length=rendering_options["box_warp"]
)
is_ray_valid = ray_end > ray_start
if torch.any(is_ray_valid).item():
ray_start[~is_ray_valid] = ray_start[is_ray_valid].min()
ray_end[~is_ray_valid] = ray_start[is_ray_valid].max()
depths_coarse = sample_stratified(
ray_origins,
ray_start,
ray_end,
rendering_options["n_samples"],
rendering_options["disparity_space_sampling"],
return_depth=return_depth,
)
else:
# Create stratified depth samples
depths_coarse = sample_stratified(
ray_origins,
rendering_options["ray_start"],
rendering_options["ray_end"],
rendering_options["n_samples"],
rendering_options["disparity_space_sampling"],
return_depth=return_depth,
)
batch_size, num_rays, samples_per_ray, _ = depths_coarse.shape
# Coarse Pass
sample_coordinates = ray_origins.unsqueeze(
-2
) + depths_coarse * ray_directions.unsqueeze(-2)
sample_directions = ray_directions.unsqueeze(-2).expand(-1, -1, samples_per_ray, -1)
return sample_coordinates, sample_directions, depths_coarse
def compute_projections(points, k_nearest_cameras):
"""
points: (B, F, (H x W), N_samples, 3)
k_nearest_cameras: (B, F, K, 32)
return:
pixel_locations: (B, F, K, (H x W), N_samples, 2)
mask: (B, F, K, (H x W), N_samples)
"""
b, f, k, _ = k_nearest_cameras.shape
intrinsics = k_nearest_cameras[..., :16].reshape(b, f, k, 4, 4) # (B, F, K, 4, 4)
c2w = k_nearest_cameras[..., 16:].reshape(b, f, k, 4, 4) # (B, F, K, 4, 4)
points_h = torch.cat(
[points, torch.ones_like(points[..., :1])], dim=-1
) # (B, F, (H x W), N_samples, 4)
points_h = repeat(points_h, "b f n s c -> b f k n s c", k=k)
w2p = torch.matmul(intrinsics, torch.inverse(c2w)) # (B, F, K, 4, 4)
projections = torch.einsum("b f k i j, b f k n s j -> b f k n s i", w2p, points_h)
# TODO: check is this correct
pixel_locations = projections[..., :2] / torch.clamp(
projections[..., 2:3], min=1e-8
)
pixel_locations = torch.clamp(
pixel_locations, min=-10, max=10
) # to avoid grid sample nan
mask = projections[..., 2] > 0 # opencv camera
inbound = (
(pixel_locations[..., 0] <= 1.0)
& (pixel_locations[..., 0] >= 0)
& (pixel_locations[..., 1] <= 1.0)
& (pixel_locations[..., 1] >= 0)
)
mask = mask & inbound
return pixel_locations, mask
class HarmonicEmbedding(torch.nn.Module):
def __init__(
self,
n_harmonic_functions: int = 6,
omega_0: float = 1.0,
logspace: bool = True,
append_input: bool = True,
) -> None:
super().__init__()
if logspace:
frequencies = 2.0 ** torch.arange(
n_harmonic_functions,
dtype=torch.float32,
)
else:
frequencies = torch.linspace(
1.0,
2.0 ** (n_harmonic_functions - 1),
n_harmonic_functions,
dtype=torch.float32,
)
self.register_buffer("_frequencies", frequencies * omega_0, persistent=False)
self.append_input = append_input
def forward(self, x: torch.Tensor) -> torch.Tensor:
"""
Args:
x: tensor of shape [..., dim]
Returns:
embedding: a harmonic embedding of `x`
of shape [..., (n_harmonic_functions * 2 + int(append_input)) * dim]
"""
embed = (x[..., None] * self._frequencies).view(*x.shape[:-1], -1)
embed = torch.cat(
(embed.sin(), embed.cos(), x)
if self.append_input
else (embed.sin(), embed.cos()),
dim=-1,
)
return embed
@staticmethod
def get_output_dim_static(
input_dims: int,
n_harmonic_functions: int,
append_input: bool,
) -> int:
"""
Utility to help predict the shape of the output of `forward`.
Args:
input_dims: length of the last dimension of the input tensor
n_harmonic_functions: number of embedding frequencies
append_input: whether or not to concat the original
input to the harmonic embedding
Returns:
int: the length of the last dimension of the output tensor
"""
return input_dims * (2 * n_harmonic_functions + int(append_input))
def get_output_dim(self, input_dims: int = 3) -> int:
"""
Same as above. The default for input_dims is 3 for 3D applications
which use harmonic embedding for positional encoding,
so the input might be xyz.
"""
return self.get_output_dim_static(
input_dims, len(self._frequencies), self.append_input
)
def encode_plucker(ray_origins, ray_dirs, harmonic_embedding=None):
"""
ray to plucker w/ pos encoding
"""
plucker = torch.cat((ray_dirs, torch.cross(ray_origins, ray_dirs, dim=-1)), dim=-1)
plucker = harmonic_embedding(plucker)
return plucker
def process_cameras(
cameras, select_ids=None, harmonic_embedding=None, rendering_options=None
):
"""
cameras: (B, F, 32)
select_ids: (B, F, K)
return:
sample_coordinates: (B, F, (H x W), N_samples, 3)
sample_directions: (B, F, (H x W), N_samples, 3)
query_plucker: (B, F, K, (H x W), 78), the plucker coordinates of the sampled rays
reference_plucker: (B, F, K, (H x W), N_samples, 78), the plucker coordinates of the sampled points
"""
_rendering_options = copy.deepcopy(rendering_options)
_rendering_options["n_samples"] = _rendering_options["n_samples"] # FIXME
resolution = _rendering_options["resolution"]
n_samples = _rendering_options["n_samples"]
b, f, c = cameras.shape
if select_ids is None:
select_ids = torch.arange(f, device=cameras.device) # (k)
select_ids = repeat(select_ids, "k -> b f k", b=b, f=f)
k_views = select_ids.shape[2]
full_cameras = repeat(cameras, "b k c -> b f k c", f=f)
k_nearest_cameras = torch.zeros(b, f, k_views, c, device=cameras.device)
b_ids = torch.arange(b)[:, None, None].expand(-1, f, k_views).to(cameras.device)
f_ids = torch.arange(f)[None, :, None].expand(b, -1, k_views).to(cameras.device)
# Use advanced indexing to get k nearest cameras
k_nearest_cameras = full_cameras[b_ids, f_ids, select_ids] # (B, F, K, 32)
instrinsics, c2w = cameras[:, :, :16], cameras[:, :, 16:]
c2w = rearrange(c2w, "b f (i j) -> (b f) i j", i=4, j=4)
instrinsics = rearrange(instrinsics, "b f (i j) -> (b f) i j", i=4, j=4)[:, :3, :3]
ray_origins, ray_directions = ray_sample(c2w, instrinsics, resolution)
sample_coordinates, sample_directions, sample_depths = sample_points(
ray_origins, ray_directions, _rendering_options, return_depth=True
)
sample_coordinates = rearrange(
sample_coordinates, "(b f) n s c -> b f n s c", f=f, n=resolution**2
) # c = 3
sample_depths = rearrange(
sample_depths, "(b f) n s c -> b f n s c", f=f, n=resolution**2
) # c = 1
sample_depths = (sample_depths - _rendering_options["ray_start"]) / (
_rendering_options["ray_end"] - _rendering_options["ray_start"]
)
projected_points, mask = compute_projections(
sample_coordinates, k_nearest_cameras
) # compute the k views projection
query_plucker = encode_plucker(
ray_origins, ray_directions, harmonic_embedding
) # (B F) N 78
query_depth = harmonic_embedding(sample_depths) # B F N S 78
query_plucker = repeat(
query_plucker, "(b f) n d -> b f k n s d", f=f, k=k_views, s=n_samples
) # B F K N 78
query_depth = repeat(query_depth, "b f n s d -> b f k n s d", k=k_views)
query_plucker = torch.cat(
(query_plucker, query_depth), dim=-1
) # B F K N S 156 # 在 i in F 帧下的深度
k_nearest_c2w = rearrange(
k_nearest_cameras[..., 16:], "b f k (i j) -> b f k i j", i=4, j=4
)
origins_cam = repeat(
k_nearest_c2w[..., :3, 3],
"b f k i -> b f k n s i",
n=resolution**2,
s=n_samples,
)
sample_coordinates = repeat(
sample_coordinates, "b f n s c -> b f k n s c", k=k_views
)
input_dirs = sample_coordinates - origins_cam # (B F K N S 3)
input_dirs = F.normalize(input_dirs, dim=-1)
canonical_w2c = compute_rel_transform(c2w, ray_origins, ray_directions)
canonical_w2c = repeat(
canonical_w2c, "(b f) n i j -> b f k n s i j", f=f, s=n_samples, k=k_views
)
origins_cam = torch.cat(
(origins_cam, torch.ones_like(origins_cam[..., :1])), dim=-1
)
cannonical_dirs = torch.einsum(
"... i j, ... j-> ... i", canonical_w2c[..., :3, :3], input_dirs
)
cannonical_cam = torch.einsum("... i j, ... j -> ... i", canonical_w2c, origins_cam)
reference_plucker = encode_plucker(
cannonical_cam, cannonical_dirs, harmonic_embedding
) # depth of the s-th point in the k-th view
reference_depth = harmonic_embedding(sample_depths) # B F N S 78
full_depth = repeat(reference_depth, "b k ... -> b f k ...", f=f)
k_nearest_depth = full_depth[b_ids, f_ids, select_ids]
reference_plucker = torch.cat(
(reference_plucker, k_nearest_depth), dim=-1
) # B F K N S 156
projected_points = projected_points * 2 - 1 # to [-1, 1]
# return projected_points, mask, (ray_origins, ray_directions), (cannonical_cam, cannonical_dirs), sample_depths
return projected_points, mask, query_plucker, reference_plucker, sample_depths