Spaces:
Sleeping
Sleeping
Download loomnet/utils/project.py from FedericoGiulio/LoomNet: direct link, hf CLI and curl.
- Browser
- Download file 18.2 kB
-
https://huggingface.co/spaces/FedericoGiulio/LoomNet/resolve/main/loomnet/utils/project.py
- Command line
-
hf download hf://spaces/FedericoGiulio/LoomNet/loomnet/utils/project.py
-
curl -L -o project.py https://huggingface.co/spaces/FedericoGiulio/LoomNet/resolve/main/loomnet/utils/project.py
18.2 kB
| from einops import rearrange, repeat | |
| import torch | |
| import torch.nn.functional as F | |
| import copy | |
| def compute_rel_transform(c2w, ray_origins, ray_directions): | |
| """ | |
| c2w: (b x f) x 4 x 4 | |
| ray_origins: (b x f) x (h x w) x 3 | |
| ray_directions: (b x f) x (h x w) x 3 | |
| return: | |
| canonical_w2c: (b x f) x (h x w) x 3 x 4 | |
| """ | |
| upv = c2w[:, :3, 1] # (b x f) x 3 | |
| upv = rearrange(upv, "bf i -> bf 1 i") # (b x f) x 1 x 3 | |
| rdotup = (ray_directions * upv).sum(-1, keepdims=True) # (b x f) x (h x w) x 1 | |
| orthoup = upv - rdotup * ray_directions # 史密斯正交化 | |
| orthoup = F.normalize(orthoup, dim=-1) # (b x f) x (h x w) x 3 | |
| vec0 = torch.cross(orthoup, ray_directions) # (b x f) x (h x w) x 3 | |
| vec0 = F.normalize(vec0, dim=-1) # (b x f) x (h x w) x 3 | |
| r_relatives = torch.stack( | |
| (vec0, orthoup, ray_directions), dim=-1 | |
| ) # (b x f) x (h x w) x 3 x 3 | |
| r_relatives_T = r_relatives.transpose(-1, -2) # (b x f) x (h x w) x 3 x 3 | |
| translation_relative = torch.einsum( | |
| "... i j, ... j -> ... i", -r_relatives_T, ray_origins | |
| ).unsqueeze( | |
| -1 | |
| ) # (b x f) x (h x w) x 3 | |
| canonical_w2c = torch.cat( | |
| (r_relatives_T, translation_relative), dim=-1 | |
| ) # (b x f) x (h x w) x 3 x 4 | |
| return canonical_w2c | |
| def ray_sample(cam2world_matrix, intrinsics, resolution): | |
| """ | |
| Create batches of rays and return origins and directions. | |
| cam2world_matrix: (N, 4, 4) | |
| intrinsics: (N, 3, 3) | |
| resolution: int | |
| ray_origins: (N, M, 3) | |
| ray_dirs: (N, M, 3) | |
| """ | |
| N, M = cam2world_matrix.shape[0], resolution**2 | |
| cam_locs_world = cam2world_matrix[:, :3, 3] | |
| fx = intrinsics[:, 0, 0] | |
| fy = intrinsics[:, 1, 1] | |
| cx = intrinsics[:, 0, 2] | |
| cy = intrinsics[:, 1, 2] | |
| sk = intrinsics[:, 0, 1] | |
| uv = torch.stack( | |
| torch.meshgrid( | |
| torch.arange( | |
| resolution, dtype=torch.float32, device=cam2world_matrix.device | |
| ), | |
| torch.arange( | |
| resolution, dtype=torch.float32, device=cam2world_matrix.device | |
| ), | |
| indexing="ij", | |
| ) | |
| ) * (1.0 / resolution) + (0.5 / resolution) | |
| uv = repeat(uv, "c h w -> b (h w) c", b=N) | |
| x_cam = uv[:, :, 0].view(N, -1) | |
| y_cam = uv[:, :, 1].view(N, -1) | |
| z_cam = torch.ones((N, M), device=cam2world_matrix.device) | |
| x_lift = ( | |
| ( | |
| x_cam | |
| - cx.unsqueeze(-1) | |
| + cy.unsqueeze(-1) * sk.unsqueeze(-1) / fy.unsqueeze(-1) | |
| - sk.unsqueeze(-1) * y_cam / fy.unsqueeze(-1) | |
| ) | |
| / fx.unsqueeze(-1) | |
| * z_cam | |
| ) | |
| y_lift = (y_cam - cy.unsqueeze(-1)) / fy.unsqueeze(-1) * z_cam | |
| cam_rel_points = torch.stack( | |
| (x_lift, y_lift, z_cam, torch.ones_like(z_cam)), dim=-1 | |
| ) | |
| world_rel_points = torch.bmm( | |
| cam2world_matrix, cam_rel_points.permute(0, 2, 1) | |
| ).permute(0, 2, 1)[:, :, :3] | |
| ray_dirs = world_rel_points - cam_locs_world[:, None, :] | |
| ray_dirs = torch.nn.functional.normalize(ray_dirs, dim=2) | |
| ray_origins = cam_locs_world.unsqueeze(1).repeat(1, ray_dirs.shape[1], 1) | |
| return ray_origins, ray_dirs | |
| def linspace(start: torch.Tensor, stop: torch.Tensor, num: int): | |
| """ | |
| Creates a tensor of shape [num, *start.shape] whose values are evenly spaced from start to end, inclusive. | |
| Replicates but the multi-dimensional bahaviour of numpy.linspace in PyTorch. | |
| """ | |
| # create a tensor of 'num' steps from 0 to 1 | |
| steps = torch.arange(num, dtype=torch.float32, device=start.device) / (num - 1) | |
| # reshape the 'steps' tensor to [-1, *([1]*start.ndim)] to allow for broadcastings | |
| # - using 'steps.reshape([-1, *([1]*start.ndim)])' would be nice here but torchscript | |
| # "cannot statically infer the expected size of a list in this contex", hence the code below | |
| for i in range(start.ndim): | |
| steps = steps.unsqueeze(-1) | |
| # the output starts at 'start' and increments until 'stop' in each dimension | |
| out = start[None] + steps * (stop - start)[None] | |
| return out | |
| def sample_stratified( | |
| ray_origins, | |
| ray_start, | |
| ray_end, | |
| depth_resolution, | |
| disparity_space_sampling=False, | |
| return_depth=False, | |
| ): | |
| """ | |
| Return depths of approximately uniformly spaced samples along rays. | |
| """ | |
| N, M, _ = ray_origins.shape | |
| if disparity_space_sampling: | |
| depths_coarse = ( | |
| torch.linspace(0, 1, depth_resolution, device=ray_origins.device) | |
| .reshape(1, 1, depth_resolution, 1) | |
| .repeat(N, M, 1, 1) | |
| ) | |
| if not return_depth: | |
| depth_delta = 1 / (depth_resolution - 1) | |
| depths_coarse += torch.rand_like(depths_coarse) * depth_delta | |
| depths_coarse = 1.0 / ( | |
| 1.0 / ray_start * (1.0 - depths_coarse) + 1.0 / ray_end * depths_coarse | |
| ) | |
| else: | |
| if type(ray_start) == torch.Tensor: | |
| depths_coarse = linspace(ray_start, ray_end, depth_resolution).permute( | |
| 1, 2, 0, 3 | |
| ) | |
| if not return_depth: | |
| depth_delta = (ray_end - ray_start) / (depth_resolution - 1) | |
| depths_coarse += torch.rand_like(depths_coarse) * depth_delta[..., None] | |
| else: | |
| depths_coarse = ( | |
| torch.linspace( | |
| ray_start, ray_end, depth_resolution, device=ray_origins.device | |
| ) | |
| .reshape(1, 1, depth_resolution, 1) | |
| .repeat(N, M, 1, 1) | |
| ) | |
| if not return_depth: | |
| depth_delta = (ray_end - ray_start) / (depth_resolution - 1) | |
| depths_coarse += torch.rand_like(depths_coarse) * depth_delta | |
| return depths_coarse | |
| def get_ray_limits_box(rays_o: torch.Tensor, rays_d: torch.Tensor, box_side_length): | |
| """ | |
| Author: Petr Kellnhofer | |
| Intersects rays with the [-1, 1] NDC volume. | |
| Returns min and max distance of entry. | |
| Returns -1 for no intersection. | |
| https://www.scratchapixel.com/lessons/3d-basic-rendering/minimal-ray-tracer-rendering-simple-shapes/ray-box-intersection | |
| """ | |
| o_shape = rays_o.shape | |
| rays_o = rays_o.detach().reshape(-1, 3) | |
| rays_d = rays_d.detach().reshape(-1, 3) | |
| bb_min = [ | |
| -1 * (box_side_length / 2), | |
| -1 * (box_side_length / 2), | |
| -1 * (box_side_length / 2), | |
| ] | |
| bb_max = [ | |
| 1 * (box_side_length / 2), | |
| 1 * (box_side_length / 2), | |
| 1 * (box_side_length / 2), | |
| ] | |
| bounds = torch.tensor([bb_min, bb_max], dtype=rays_o.dtype, device=rays_o.device) | |
| is_valid = torch.ones(rays_o.shape[:-1], dtype=bool, device=rays_o.device) | |
| # Precompute inverse for stability. | |
| invdir = 1 / rays_d | |
| sign = (invdir < 0).long() | |
| # Intersect with YZ plane. | |
| tmin = (bounds.index_select(0, sign[..., 0])[..., 0] - rays_o[..., 0]) * invdir[ | |
| ..., 0 | |
| ] | |
| tmax = (bounds.index_select(0, 1 - sign[..., 0])[..., 0] - rays_o[..., 0]) * invdir[ | |
| ..., 0 | |
| ] | |
| # Intersect with XZ plane. | |
| tymin = (bounds.index_select(0, sign[..., 1])[..., 1] - rays_o[..., 1]) * invdir[ | |
| ..., 1 | |
| ] | |
| tymax = ( | |
| bounds.index_select(0, 1 - sign[..., 1])[..., 1] - rays_o[..., 1] | |
| ) * invdir[..., 1] | |
| # Resolve parallel rays. | |
| is_valid[torch.logical_or(tmin > tymax, tymin > tmax)] = False | |
| # Use the shortest intersection. | |
| tmin = torch.max(tmin, tymin) | |
| tmax = torch.min(tmax, tymax) | |
| # Intersect with XY plane. | |
| tzmin = (bounds.index_select(0, sign[..., 2])[..., 2] - rays_o[..., 2]) * invdir[ | |
| ..., 2 | |
| ] | |
| tzmax = ( | |
| bounds.index_select(0, 1 - sign[..., 2])[..., 2] - rays_o[..., 2] | |
| ) * invdir[..., 2] | |
| # Resolve parallel rays. | |
| is_valid[torch.logical_or(tmin > tzmax, tzmin > tmax)] = False | |
| # Use the shortest intersection. | |
| tmin = torch.max(tmin, tzmin) | |
| tmax = torch.min(tmax, tzmax) | |
| # Mark invalid. | |
| tmin[torch.logical_not(is_valid)] = -1 | |
| tmax[torch.logical_not(is_valid)] = -2 | |
| return tmin.reshape(*o_shape[:-1], 1), tmax.reshape(*o_shape[:-1], 1) | |
| def sample_points(ray_origins, ray_directions, rendering_options, return_depth=False): | |
| if rendering_options["ray_start"] == rendering_options["ray_end"] == "auto": | |
| ray_start, ray_end = get_ray_limits_box( | |
| ray_origins, ray_directions, box_side_length=rendering_options["box_warp"] | |
| ) | |
| is_ray_valid = ray_end > ray_start | |
| if torch.any(is_ray_valid).item(): | |
| ray_start[~is_ray_valid] = ray_start[is_ray_valid].min() | |
| ray_end[~is_ray_valid] = ray_start[is_ray_valid].max() | |
| depths_coarse = sample_stratified( | |
| ray_origins, | |
| ray_start, | |
| ray_end, | |
| rendering_options["n_samples"], | |
| rendering_options["disparity_space_sampling"], | |
| return_depth=return_depth, | |
| ) | |
| else: | |
| # Create stratified depth samples | |
| depths_coarse = sample_stratified( | |
| ray_origins, | |
| rendering_options["ray_start"], | |
| rendering_options["ray_end"], | |
| rendering_options["n_samples"], | |
| rendering_options["disparity_space_sampling"], | |
| return_depth=return_depth, | |
| ) | |
| batch_size, num_rays, samples_per_ray, _ = depths_coarse.shape | |
| # Coarse Pass | |
| sample_coordinates = ray_origins.unsqueeze( | |
| -2 | |
| ) + depths_coarse * ray_directions.unsqueeze(-2) | |
| sample_directions = ray_directions.unsqueeze(-2).expand(-1, -1, samples_per_ray, -1) | |
| return sample_coordinates, sample_directions, depths_coarse | |
| def compute_projections(points, k_nearest_cameras): | |
| """ | |
| points: (B, F, (H x W), N_samples, 3) | |
| k_nearest_cameras: (B, F, K, 32) | |
| return: | |
| pixel_locations: (B, F, K, (H x W), N_samples, 2) | |
| mask: (B, F, K, (H x W), N_samples) | |
| """ | |
| b, f, k, _ = k_nearest_cameras.shape | |
| intrinsics = k_nearest_cameras[..., :16].reshape(b, f, k, 4, 4) # (B, F, K, 4, 4) | |
| c2w = k_nearest_cameras[..., 16:].reshape(b, f, k, 4, 4) # (B, F, K, 4, 4) | |
| points_h = torch.cat( | |
| [points, torch.ones_like(points[..., :1])], dim=-1 | |
| ) # (B, F, (H x W), N_samples, 4) | |
| points_h = repeat(points_h, "b f n s c -> b f k n s c", k=k) | |
| w2p = torch.matmul(intrinsics, torch.inverse(c2w)) # (B, F, K, 4, 4) | |
| projections = torch.einsum("b f k i j, b f k n s j -> b f k n s i", w2p, points_h) | |
| # TODO: check is this correct | |
| pixel_locations = projections[..., :2] / torch.clamp( | |
| projections[..., 2:3], min=1e-8 | |
| ) | |
| pixel_locations = torch.clamp( | |
| pixel_locations, min=-10, max=10 | |
| ) # to avoid grid sample nan | |
| mask = projections[..., 2] > 0 # opencv camera | |
| inbound = ( | |
| (pixel_locations[..., 0] <= 1.0) | |
| & (pixel_locations[..., 0] >= 0) | |
| & (pixel_locations[..., 1] <= 1.0) | |
| & (pixel_locations[..., 1] >= 0) | |
| ) | |
| mask = mask & inbound | |
| return pixel_locations, mask | |
| class HarmonicEmbedding(torch.nn.Module): | |
| def __init__( | |
| self, | |
| n_harmonic_functions: int = 6, | |
| omega_0: float = 1.0, | |
| logspace: bool = True, | |
| append_input: bool = True, | |
| ) -> None: | |
| super().__init__() | |
| if logspace: | |
| frequencies = 2.0 ** torch.arange( | |
| n_harmonic_functions, | |
| dtype=torch.float32, | |
| ) | |
| else: | |
| frequencies = torch.linspace( | |
| 1.0, | |
| 2.0 ** (n_harmonic_functions - 1), | |
| n_harmonic_functions, | |
| dtype=torch.float32, | |
| ) | |
| self.register_buffer("_frequencies", frequencies * omega_0, persistent=False) | |
| self.append_input = append_input | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| """ | |
| Args: | |
| x: tensor of shape [..., dim] | |
| Returns: | |
| embedding: a harmonic embedding of `x` | |
| of shape [..., (n_harmonic_functions * 2 + int(append_input)) * dim] | |
| """ | |
| embed = (x[..., None] * self._frequencies).view(*x.shape[:-1], -1) | |
| embed = torch.cat( | |
| (embed.sin(), embed.cos(), x) | |
| if self.append_input | |
| else (embed.sin(), embed.cos()), | |
| dim=-1, | |
| ) | |
| return embed | |
| def get_output_dim_static( | |
| input_dims: int, | |
| n_harmonic_functions: int, | |
| append_input: bool, | |
| ) -> int: | |
| """ | |
| Utility to help predict the shape of the output of `forward`. | |
| Args: | |
| input_dims: length of the last dimension of the input tensor | |
| n_harmonic_functions: number of embedding frequencies | |
| append_input: whether or not to concat the original | |
| input to the harmonic embedding | |
| Returns: | |
| int: the length of the last dimension of the output tensor | |
| """ | |
| return input_dims * (2 * n_harmonic_functions + int(append_input)) | |
| def get_output_dim(self, input_dims: int = 3) -> int: | |
| """ | |
| Same as above. The default for input_dims is 3 for 3D applications | |
| which use harmonic embedding for positional encoding, | |
| so the input might be xyz. | |
| """ | |
| return self.get_output_dim_static( | |
| input_dims, len(self._frequencies), self.append_input | |
| ) | |
| def encode_plucker(ray_origins, ray_dirs, harmonic_embedding=None): | |
| """ | |
| ray to plucker w/ pos encoding | |
| """ | |
| plucker = torch.cat((ray_dirs, torch.cross(ray_origins, ray_dirs, dim=-1)), dim=-1) | |
| plucker = harmonic_embedding(plucker) | |
| return plucker | |
| def process_cameras( | |
| cameras, select_ids=None, harmonic_embedding=None, rendering_options=None | |
| ): | |
| """ | |
| cameras: (B, F, 32) | |
| select_ids: (B, F, K) | |
| return: | |
| sample_coordinates: (B, F, (H x W), N_samples, 3) | |
| sample_directions: (B, F, (H x W), N_samples, 3) | |
| query_plucker: (B, F, K, (H x W), 78), the plucker coordinates of the sampled rays | |
| reference_plucker: (B, F, K, (H x W), N_samples, 78), the plucker coordinates of the sampled points | |
| """ | |
| _rendering_options = copy.deepcopy(rendering_options) | |
| _rendering_options["n_samples"] = _rendering_options["n_samples"] # FIXME | |
| resolution = _rendering_options["resolution"] | |
| n_samples = _rendering_options["n_samples"] | |
| b, f, c = cameras.shape | |
| if select_ids is None: | |
| select_ids = torch.arange(f, device=cameras.device) # (k) | |
| select_ids = repeat(select_ids, "k -> b f k", b=b, f=f) | |
| k_views = select_ids.shape[2] | |
| full_cameras = repeat(cameras, "b k c -> b f k c", f=f) | |
| k_nearest_cameras = torch.zeros(b, f, k_views, c, device=cameras.device) | |
| b_ids = torch.arange(b)[:, None, None].expand(-1, f, k_views).to(cameras.device) | |
| f_ids = torch.arange(f)[None, :, None].expand(b, -1, k_views).to(cameras.device) | |
| # Use advanced indexing to get k nearest cameras | |
| k_nearest_cameras = full_cameras[b_ids, f_ids, select_ids] # (B, F, K, 32) | |
| instrinsics, c2w = cameras[:, :, :16], cameras[:, :, 16:] | |
| c2w = rearrange(c2w, "b f (i j) -> (b f) i j", i=4, j=4) | |
| instrinsics = rearrange(instrinsics, "b f (i j) -> (b f) i j", i=4, j=4)[:, :3, :3] | |
| ray_origins, ray_directions = ray_sample(c2w, instrinsics, resolution) | |
| sample_coordinates, sample_directions, sample_depths = sample_points( | |
| ray_origins, ray_directions, _rendering_options, return_depth=True | |
| ) | |
| sample_coordinates = rearrange( | |
| sample_coordinates, "(b f) n s c -> b f n s c", f=f, n=resolution**2 | |
| ) # c = 3 | |
| sample_depths = rearrange( | |
| sample_depths, "(b f) n s c -> b f n s c", f=f, n=resolution**2 | |
| ) # c = 1 | |
| sample_depths = (sample_depths - _rendering_options["ray_start"]) / ( | |
| _rendering_options["ray_end"] - _rendering_options["ray_start"] | |
| ) | |
| projected_points, mask = compute_projections( | |
| sample_coordinates, k_nearest_cameras | |
| ) # compute the k views projection | |
| query_plucker = encode_plucker( | |
| ray_origins, ray_directions, harmonic_embedding | |
| ) # (B F) N 78 | |
| query_depth = harmonic_embedding(sample_depths) # B F N S 78 | |
| query_plucker = repeat( | |
| query_plucker, "(b f) n d -> b f k n s d", f=f, k=k_views, s=n_samples | |
| ) # B F K N 78 | |
| query_depth = repeat(query_depth, "b f n s d -> b f k n s d", k=k_views) | |
| query_plucker = torch.cat( | |
| (query_plucker, query_depth), dim=-1 | |
| ) # B F K N S 156 # 在 i in F 帧下的深度 | |
| k_nearest_c2w = rearrange( | |
| k_nearest_cameras[..., 16:], "b f k (i j) -> b f k i j", i=4, j=4 | |
| ) | |
| origins_cam = repeat( | |
| k_nearest_c2w[..., :3, 3], | |
| "b f k i -> b f k n s i", | |
| n=resolution**2, | |
| s=n_samples, | |
| ) | |
| sample_coordinates = repeat( | |
| sample_coordinates, "b f n s c -> b f k n s c", k=k_views | |
| ) | |
| input_dirs = sample_coordinates - origins_cam # (B F K N S 3) | |
| input_dirs = F.normalize(input_dirs, dim=-1) | |
| canonical_w2c = compute_rel_transform(c2w, ray_origins, ray_directions) | |
| canonical_w2c = repeat( | |
| canonical_w2c, "(b f) n i j -> b f k n s i j", f=f, s=n_samples, k=k_views | |
| ) | |
| origins_cam = torch.cat( | |
| (origins_cam, torch.ones_like(origins_cam[..., :1])), dim=-1 | |
| ) | |
| cannonical_dirs = torch.einsum( | |
| "... i j, ... j-> ... i", canonical_w2c[..., :3, :3], input_dirs | |
| ) | |
| cannonical_cam = torch.einsum("... i j, ... j -> ... i", canonical_w2c, origins_cam) | |
| reference_plucker = encode_plucker( | |
| cannonical_cam, cannonical_dirs, harmonic_embedding | |
| ) # depth of the s-th point in the k-th view | |
| reference_depth = harmonic_embedding(sample_depths) # B F N S 78 | |
| full_depth = repeat(reference_depth, "b k ... -> b f k ...", f=f) | |
| k_nearest_depth = full_depth[b_ids, f_ids, select_ids] | |
| reference_plucker = torch.cat( | |
| (reference_plucker, k_nearest_depth), dim=-1 | |
| ) # B F K N S 156 | |
| projected_points = projected_points * 2 - 1 # to [-1, 1] | |
| # return projected_points, mask, (ray_origins, ray_directions), (cannonical_cam, cannonical_dirs), sample_depths | |
| return projected_points, mask, query_plucker, reference_plucker, sample_depths | |