diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/ext.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/ext.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6a17dbb0ce77bc13d90003889c0abff853b09be6 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/ext.cpp @@ -0,0 +1,186 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +// clang-format off +#include "./pulsar/global.h" // Include before . +#include +// clang-format on +#include "./pulsar/pytorch/renderer.h" +#include "./pulsar/pytorch/tensor_util.h" +#include "ball_query/ball_query.h" +#include "blending/sigmoid_alpha_blend.h" +#include "compositing/alpha_composite.h" +#include "compositing/norm_weighted_sum.h" +#include "compositing/weighted_sum.h" +#include "face_areas_normals/face_areas_normals.h" +#include "gather_scatter/gather_scatter.h" +#include "interp_face_attrs/interp_face_attrs.h" +#include "iou_box3d/iou_box3d.h" +#include "knn/knn.h" +#include "marching_cubes/marching_cubes.h" +#include "mesh_normal_consistency/mesh_normal_consistency.h" +#include "packed_to_padded_tensor/packed_to_padded_tensor.h" +#include "point_mesh/point_mesh_cuda.h" +#include "points_to_volumes/points_to_volumes.h" +#include "rasterize_meshes/rasterize_meshes.h" +#include "rasterize_points/rasterize_points.h" +#include "sample_farthest_points/sample_farthest_points.h" +#include "sample_pdf/sample_pdf.h" + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("face_areas_normals_forward", &FaceAreasNormalsForward); + m.def("face_areas_normals_backward", &FaceAreasNormalsBackward); + m.def("packed_to_padded", &PackedToPadded); + m.def("padded_to_packed", &PaddedToPacked); + m.def("interp_face_attrs_forward", &InterpFaceAttrsForward); + m.def("interp_face_attrs_backward", &InterpFaceAttrsBackward); +#ifdef WITH_CUDA + m.def("knn_check_version", &KnnCheckVersion); +#endif + m.def("knn_points_idx", &KNearestNeighborIdx); + m.def("knn_points_backward", &KNearestNeighborBackward); + m.def("ball_query", &BallQuery); + m.def("sample_farthest_points", &FarthestPointSampling); + m.def( + "mesh_normal_consistency_find_verts", &MeshNormalConsistencyFindVertices); + m.def("gather_scatter", &GatherScatter); + m.def("points_to_volumes_forward", PointsToVolumesForward); + m.def("points_to_volumes_backward", PointsToVolumesBackward); + m.def("rasterize_points", &RasterizePoints); + m.def("rasterize_points_backward", &RasterizePointsBackward); + m.def("rasterize_meshes_backward", &RasterizeMeshesBackward); + m.def("rasterize_meshes", &RasterizeMeshes); + m.def("sigmoid_alpha_blend", &SigmoidAlphaBlend); + m.def("sigmoid_alpha_blend_backward", &SigmoidAlphaBlendBackward); + + // Accumulation functions + m.def("accum_weightedsumnorm", &weightedSumNormForward); + m.def("accum_weightedsum", &weightedSumForward); + m.def("accum_alphacomposite", &alphaCompositeForward); + m.def("accum_weightedsumnorm_backward", &weightedSumNormBackward); + m.def("accum_weightedsum_backward", &weightedSumBackward); + m.def("accum_alphacomposite_backward", &alphaCompositeBackward); + + // These are only visible for testing; users should not call them directly + m.def("_rasterize_points_coarse", &RasterizePointsCoarse); + m.def("_rasterize_points_naive", &RasterizePointsNaive); + m.def("_rasterize_meshes_naive", &RasterizeMeshesNaive); + m.def("_rasterize_meshes_coarse", &RasterizeMeshesCoarse); + m.def("_rasterize_meshes_fine", &RasterizeMeshesFine); + + // PointEdge distance functions + m.def("point_edge_dist_forward", &PointEdgeDistanceForward); + m.def("point_edge_dist_backward", &PointEdgeDistanceBackward); + m.def("edge_point_dist_forward", &EdgePointDistanceForward); + m.def("edge_point_dist_backward", &EdgePointDistanceBackward); + m.def("point_edge_array_dist_forward", &PointEdgeArrayDistanceForward); + m.def("point_edge_array_dist_backward", &PointEdgeArrayDistanceBackward); + + // PointFace distance functions + m.def("point_face_dist_forward", &PointFaceDistanceForward); + m.def("point_face_dist_backward", &PointFaceDistanceBackward); + m.def("face_point_dist_forward", &FacePointDistanceForward); + m.def("face_point_dist_backward", &FacePointDistanceBackward); + m.def("point_face_array_dist_forward", &PointFaceArrayDistanceForward); + m.def("point_face_array_dist_backward", &PointFaceArrayDistanceBackward); + + // Sample PDF + m.def("sample_pdf", &SamplePdf); + + // 3D IoU + m.def("iou_box3d", &IoUBox3D); + + // Marching cubes + m.def("marching_cubes", &MarchingCubes); + + // Pulsar. +#ifdef PULSAR_LOGGING_ENABLED + c10::ShowLogInfoToStderr(); +#endif + py::class_< + pulsar::pytorch::Renderer, + std::shared_ptr>(m, "PulsarRenderer") + .def(py::init< + const uint&, + const uint&, + const uint&, + const bool&, + const bool&, + const float&, + const uint&, + const uint&>()) + .def( + "__eq__", + [](const pulsar::pytorch::Renderer& a, + const pulsar::pytorch::Renderer& b) { return a == b; }, + py::is_operator()) + .def( + "__ne__", + [](const pulsar::pytorch::Renderer& a, + const pulsar::pytorch::Renderer& b) { return !(a == b); }, + py::is_operator()) + .def( + "__repr__", + [](const pulsar::pytorch::Renderer& self) { + std::stringstream ss; + ss << self; + return ss.str(); + }) + .def( + "forward", + &pulsar::pytorch::Renderer::forward, + py::arg("vert_pos"), + py::arg("vert_col"), + py::arg("vert_radii"), + + py::arg("cam_pos"), + py::arg("pixel_0_0_center"), + py::arg("pixel_vec_x"), + py::arg("pixel_vec_y"), + py::arg("focal_length"), + py::arg("principal_point_offsets"), + + py::arg("gamma"), + py::arg("max_depth"), + py::arg("min_depth") /* = 0.f*/, + py::arg( + "bg_col") /* = at::nullopt not exposed properly in pytorch 1.1. */ + , + py::arg("opacity") /* = at::nullopt ... */, + py::arg("percent_allowed_difference") = 0.01f, + py::arg("max_n_hits") = MAX_UINT, + py::arg("mode") = 0) + .def("backward", &pulsar::pytorch::Renderer::backward) + .def_property( + "device_tracker", + [](const pulsar::pytorch::Renderer& self) { + return self.device_tracker; + }, + [](pulsar::pytorch::Renderer& self, const torch::Tensor& val) { + self.device_tracker = val; + }) + .def_property_readonly("width", &pulsar::pytorch::Renderer::width) + .def_property_readonly("height", &pulsar::pytorch::Renderer::height) + .def_property_readonly( + "max_num_balls", &pulsar::pytorch::Renderer::max_num_balls) + .def_property_readonly( + "orthogonal", &pulsar::pytorch::Renderer::orthogonal) + .def_property_readonly( + "right_handed", &pulsar::pytorch::Renderer::right_handed) + .def_property_readonly("n_track", &pulsar::pytorch::Renderer::n_track); + m.def( + "pulsar_sphere_ids_from_result_info_nograd", + &pulsar::pytorch::sphere_ids_from_result_info_nograd); + // Constants. + m.attr("EPS") = py::float_(EPS); + m.attr("MAX_FLOAT") = py::float_(MAX_FLOAT); + m.attr("MAX_INT") = py::int_(MAX_INT); + m.attr("MAX_UINT") = py::int_(MAX_UINT); + m.attr("MAX_USHORT") = py::int_(MAX_USHORT); + m.attr("PULSAR_MAX_GRAD_SPHERES") = py::int_(MAX_GRAD_SPHERES); +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/marching_cubes/tables.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/marching_cubes/tables.h new file mode 100644 index 0000000000000000000000000000000000000000..3aff617c53e5ba963c014cabd7beb1c2cd61a053 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/marching_cubes/tables.h @@ -0,0 +1,294 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once +using uint = unsigned int; + +// A table mapping from cubeindex to a list of face configurations. +// Each list contains at most 5 faces, where each face is represented with +// 3 consecutive numbers +// Table adapted from http://paulbourke.net/geometry/polygonise/ +// +#define X 255 +const unsigned char _FACE_TABLE[256][16] = { + {X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X}, + {0, 8, 3, X, X, X, X, X, X, X, X, X, X, X, X, X}, + {0, 1, 9, X, X, X, X, X, X, X, X, X, X, X, X, X}, + {1, 8, 3, 9, 8, 1, X, X, X, X, X, X, X, X, X, X}, + {1, 2, 10, X, X, X, X, X, X, X, X, X, X, X, X, X}, + {0, 8, 3, 1, 2, 10, X, X, X, X, X, X, X, X, X, X}, + {9, 2, 10, 0, 2, 9, X, X, X, X, X, X, X, X, X, X}, + {2, 8, 3, 2, 10, 8, 10, 9, 8, X, X, X, X, X, X, X}, + {3, 11, 2, X, X, X, X, X, X, X, X, X, X, X, X, X}, + {0, 11, 2, 8, 11, 0, X, X, X, X, X, X, X, X, X, X}, + {1, 9, 0, 2, 3, 11, X, X, X, X, X, X, X, X, X, X}, + {1, 11, 2, 1, 9, 11, 9, 8, 11, X, X, X, X, X, X, X}, + {3, 10, 1, 11, 10, 3, X, X, X, X, X, X, X, X, X, X}, + {0, 10, 1, 0, 8, 10, 8, 11, 10, X, X, X, X, X, X, X}, + {3, 9, 0, 3, 11, 9, 11, 10, 9, X, X, X, X, X, X, X}, + {9, 8, 10, 10, 8, 11, X, X, X, X, X, X, X, X, X, X}, + {4, 7, 8, X, X, X, X, X, X, X, X, X, X, X, X, X}, + {4, 3, 0, 7, 3, 4, X, X, X, X, X, X, X, X, X, X}, + {0, 1, 9, 8, 4, 7, X, X, X, X, X, X, X, X, X, X}, + {4, 1, 9, 4, 7, 1, 7, 3, 1, X, X, X, X, X, X, X}, + {1, 2, 10, 8, 4, 7, X, X, X, X, X, X, X, X, X, X}, + {3, 4, 7, 3, 0, 4, 1, 2, 10, X, X, X, X, X, X, X}, + {9, 2, 10, 9, 0, 2, 8, 4, 7, X, X, X, X, X, X, X}, + {2, 10, 9, 2, 9, 7, 2, 7, 3, 7, 9, 4, X, X, X, X}, + {8, 4, 7, 3, 11, 2, X, X, X, X, X, X, X, X, X, X}, + {11, 4, 7, 11, 2, 4, 2, 0, 4, X, X, X, X, X, X, X}, + {9, 0, 1, 8, 4, 7, 2, 3, 11, X, X, X, X, X, X, X}, + {4, 7, 11, 9, 4, 11, 9, 11, 2, 9, 2, 1, X, X, X, X}, + {3, 10, 1, 3, 11, 10, 7, 8, 4, X, X, X, X, X, X, X}, + {1, 11, 10, 1, 4, 11, 1, 0, 4, 7, 11, 4, X, X, X, X}, + {4, 7, 8, 9, 0, 11, 9, 11, 10, 11, 0, 3, X, X, X, X}, + {4, 7, 11, 4, 11, 9, 9, 11, 10, X, X, X, X, X, X, X}, + {9, 5, 4, X, X, X, X, X, X, X, X, X, X, X, X, X}, + {9, 5, 4, 0, 8, 3, X, X, X, X, X, X, X, X, X, X}, + {0, 5, 4, 1, 5, 0, X, X, X, X, X, X, X, X, X, X}, + {8, 5, 4, 8, 3, 5, 3, 1, 5, X, X, X, X, X, X, X}, + {1, 2, 10, 9, 5, 4, X, X, X, X, X, X, X, X, X, X}, + {3, 0, 8, 1, 2, 10, 4, 9, 5, X, X, X, X, X, X, X}, + {5, 2, 10, 5, 4, 2, 4, 0, 2, X, X, X, X, X, X, X}, + {2, 10, 5, 3, 2, 5, 3, 5, 4, 3, 4, 8, X, X, X, X}, + {9, 5, 4, 2, 3, 11, X, X, X, X, X, X, X, X, X, X}, + {0, 11, 2, 0, 8, 11, 4, 9, 5, X, X, X, X, X, X, X}, + {0, 5, 4, 0, 1, 5, 2, 3, 11, X, X, X, X, X, X, X}, + {2, 1, 5, 2, 5, 8, 2, 8, 11, 4, 8, 5, X, X, X, X}, + {10, 3, 11, 10, 1, 3, 9, 5, 4, X, X, X, X, X, X, X}, + {4, 9, 5, 0, 8, 1, 8, 10, 1, 8, 11, 10, X, X, X, X}, + {5, 4, 0, 5, 0, 11, 5, 11, 10, 11, 0, 3, X, X, X, X}, + {5, 4, 8, 5, 8, 10, 10, 8, 11, X, X, X, X, X, X, X}, + {9, 7, 8, 5, 7, 9, X, X, X, X, X, X, X, X, X, X}, + {9, 3, 0, 9, 5, 3, 5, 7, 3, X, X, X, X, X, X, X}, + {0, 7, 8, 0, 1, 7, 1, 5, 7, X, X, X, X, X, X, X}, + {1, 5, 3, 3, 5, 7, X, X, X, X, X, X, X, X, X, X}, + {9, 7, 8, 9, 5, 7, 10, 1, 2, X, X, X, X, X, X, X}, + {10, 1, 2, 9, 5, 0, 5, 3, 0, 5, 7, 3, X, X, X, X}, + {8, 0, 2, 8, 2, 5, 8, 5, 7, 10, 5, 2, X, X, X, X}, + {2, 10, 5, 2, 5, 3, 3, 5, 7, X, X, X, X, X, X, X}, + {7, 9, 5, 7, 8, 9, 3, 11, 2, X, X, X, X, X, X, X}, + {9, 5, 7, 9, 7, 2, 9, 2, 0, 2, 7, 11, X, X, X, X}, + {2, 3, 11, 0, 1, 8, 1, 7, 8, 1, 5, 7, X, X, X, X}, + {11, 2, 1, 11, 1, 7, 7, 1, 5, X, X, X, X, X, X, X}, + {9, 5, 8, 8, 5, 7, 10, 1, 3, 10, 3, 11, X, X, X, X}, + {5, 7, 0, 5, 0, 9, 7, 11, 0, 1, 0, 10, 11, 10, 0, X}, + {11, 10, 0, 11, 0, 3, 10, 5, 0, 8, 0, 7, 5, 7, 0, X}, + {11, 10, 5, 7, 11, 5, X, X, X, X, X, X, X, X, X, X}, + {10, 6, 5, X, X, X, X, X, X, X, X, X, X, X, X, X}, + {0, 8, 3, 5, 10, 6, X, X, X, X, X, X, X, X, X, X}, + {9, 0, 1, 5, 10, 6, X, X, X, X, X, X, X, X, X, X}, + {1, 8, 3, 1, 9, 8, 5, 10, 6, X, X, X, X, X, X, X}, + {1, 6, 5, 2, 6, 1, X, X, X, X, X, X, X, X, X, X}, + {1, 6, 5, 1, 2, 6, 3, 0, 8, X, X, X, X, X, X, X}, + {9, 6, 5, 9, 0, 6, 0, 2, 6, X, X, X, X, X, X, X}, + {5, 9, 8, 5, 8, 2, 5, 2, 6, 3, 2, 8, X, X, X, X}, + {2, 3, 11, 10, 6, 5, X, X, X, X, X, X, X, X, X, X}, + {11, 0, 8, 11, 2, 0, 10, 6, 5, X, X, X, X, X, X, X}, + {0, 1, 9, 2, 3, 11, 5, 10, 6, X, X, X, X, X, X, X}, + {5, 10, 6, 1, 9, 2, 9, 11, 2, 9, 8, 11, X, X, X, X}, + {6, 3, 11, 6, 5, 3, 5, 1, 3, X, X, X, X, X, X, X}, + {0, 8, 11, 0, 11, 5, 0, 5, 1, 5, 11, 6, X, X, X, X}, + {3, 11, 6, 0, 3, 6, 0, 6, 5, 0, 5, 9, X, X, X, X}, + {6, 5, 9, 6, 9, 11, 11, 9, 8, X, X, X, X, X, X, X}, + {5, 10, 6, 4, 7, 8, X, X, X, X, X, X, X, X, X, X}, + {4, 3, 0, 4, 7, 3, 6, 5, 10, X, X, X, X, X, X, X}, + {1, 9, 0, 5, 10, 6, 8, 4, 7, X, X, X, X, X, X, X}, + {10, 6, 5, 1, 9, 7, 1, 7, 3, 7, 9, 4, X, X, X, X}, + {6, 1, 2, 6, 5, 1, 4, 7, 8, X, X, X, X, X, X, X}, + {1, 2, 5, 5, 2, 6, 3, 0, 4, 3, 4, 7, X, X, X, X}, + {8, 4, 7, 9, 0, 5, 0, 6, 5, 0, 2, 6, X, X, X, X}, + {7, 3, 9, 7, 9, 4, 3, 2, 9, 5, 9, 6, 2, 6, 9, X}, + {3, 11, 2, 7, 8, 4, 10, 6, 5, X, X, X, X, X, X, X}, + {5, 10, 6, 4, 7, 2, 4, 2, 0, 2, 7, 11, X, X, X, X}, + {0, 1, 9, 4, 7, 8, 2, 3, 11, 5, 10, 6, X, X, X, X}, + {9, 2, 1, 9, 11, 2, 9, 4, 11, 7, 11, 4, 5, 10, 6, X}, + {8, 4, 7, 3, 11, 5, 3, 5, 1, 5, 11, 6, X, X, X, X}, + {5, 1, 11, 5, 11, 6, 1, 0, 11, 7, 11, 4, 0, 4, 11, X}, + {0, 5, 9, 0, 6, 5, 0, 3, 6, 11, 6, 3, 8, 4, 7, X}, + {6, 5, 9, 6, 9, 11, 4, 7, 9, 7, 11, 9, X, X, X, X}, + {10, 4, 9, 6, 4, 10, X, X, X, X, X, X, X, X, X, X}, + {4, 10, 6, 4, 9, 10, 0, 8, 3, X, X, X, X, X, X, X}, + {10, 0, 1, 10, 6, 0, 6, 4, 0, X, X, X, X, X, X, X}, + {8, 3, 1, 8, 1, 6, 8, 6, 4, 6, 1, 10, X, X, X, X}, + {1, 4, 9, 1, 2, 4, 2, 6, 4, X, X, X, X, X, X, X}, + {3, 0, 8, 1, 2, 9, 2, 4, 9, 2, 6, 4, X, X, X, X}, + {0, 2, 4, 4, 2, 6, X, X, X, X, X, X, X, X, X, X}, + {8, 3, 2, 8, 2, 4, 4, 2, 6, X, X, X, X, X, X, X}, + {10, 4, 9, 10, 6, 4, 11, 2, 3, X, X, X, X, X, X, X}, + {0, 8, 2, 2, 8, 11, 4, 9, 10, 4, 10, 6, X, X, X, X}, + {3, 11, 2, 0, 1, 6, 0, 6, 4, 6, 1, 10, X, X, X, X}, + {6, 4, 1, 6, 1, 10, 4, 8, 1, 2, 1, 11, 8, 11, 1, X}, + {9, 6, 4, 9, 3, 6, 9, 1, 3, 11, 6, 3, X, X, X, X}, + {8, 11, 1, 8, 1, 0, 11, 6, 1, 9, 1, 4, 6, 4, 1, X}, + {3, 11, 6, 3, 6, 0, 0, 6, 4, X, X, X, X, X, X, X}, + {6, 4, 8, 11, 6, 8, X, X, X, X, X, X, X, X, X, X}, + {7, 10, 6, 7, 8, 10, 8, 9, 10, X, X, X, X, X, X, X}, + {0, 7, 3, 0, 10, 7, 0, 9, 10, 6, 7, 10, X, X, X, X}, + {10, 6, 7, 1, 10, 7, 1, 7, 8, 1, 8, 0, X, X, X, X}, + {10, 6, 7, 10, 7, 1, 1, 7, 3, X, X, X, X, X, X, X}, + {1, 2, 6, 1, 6, 8, 1, 8, 9, 8, 6, 7, X, X, X, X}, + {2, 6, 9, 2, 9, 1, 6, 7, 9, 0, 9, 3, 7, 3, 9, X}, + {7, 8, 0, 7, 0, 6, 6, 0, 2, X, X, X, X, X, X, X}, + {7, 3, 2, 6, 7, 2, X, X, X, X, X, X, X, X, X, X}, + {2, 3, 11, 10, 6, 8, 10, 8, 9, 8, 6, 7, X, X, X, X}, + {2, 0, 7, 2, 7, 11, 0, 9, 7, 6, 7, 10, 9, 10, 7, X}, + {1, 8, 0, 1, 7, 8, 1, 10, 7, 6, 7, 10, 2, 3, 11, X}, + {11, 2, 1, 11, 1, 7, 10, 6, 1, 6, 7, 1, X, X, X, X}, + {8, 9, 6, 8, 6, 7, 9, 1, 6, 11, 6, 3, 1, 3, 6, X}, + {0, 9, 1, 11, 6, 7, X, X, X, X, X, X, X, X, X, X}, + {7, 8, 0, 7, 0, 6, 3, 11, 0, 11, 6, 0, X, X, X, X}, + {7, 11, 6, X, X, X, X, X, X, X, X, X, X, X, X, X}, + {7, 6, 11, X, X, X, X, X, X, X, X, X, X, X, X, X}, + {3, 0, 8, 11, 7, 6, X, X, X, X, X, X, X, X, X, X}, + {0, 1, 9, 11, 7, 6, X, X, X, X, X, X, X, X, X, X}, + {8, 1, 9, 8, 3, 1, 11, 7, 6, X, X, X, X, X, X, X}, + {10, 1, 2, 6, 11, 7, X, X, X, X, X, X, X, X, X, X}, + {1, 2, 10, 3, 0, 8, 6, 11, 7, X, X, X, X, X, X, X}, + {2, 9, 0, 2, 10, 9, 6, 11, 7, X, X, X, X, X, X, X}, + {6, 11, 7, 2, 10, 3, 10, 8, 3, 10, 9, 8, X, X, X, X}, + {7, 2, 3, 6, 2, 7, X, X, X, X, X, X, X, X, X, X}, + {7, 0, 8, 7, 6, 0, 6, 2, 0, X, X, X, X, X, X, X}, + {2, 7, 6, 2, 3, 7, 0, 1, 9, X, X, X, X, X, X, X}, + {1, 6, 2, 1, 8, 6, 1, 9, 8, 8, 7, 6, X, X, X, X}, + {10, 7, 6, 10, 1, 7, 1, 3, 7, X, X, X, X, X, X, X}, + {10, 7, 6, 1, 7, 10, 1, 8, 7, 1, 0, 8, X, X, X, X}, + {0, 3, 7, 0, 7, 10, 0, 10, 9, 6, 10, 7, X, X, X, X}, + {7, 6, 10, 7, 10, 8, 8, 10, 9, X, X, X, X, X, X, X}, + {6, 8, 4, 11, 8, 6, X, X, X, X, X, X, X, X, X, X}, + {3, 6, 11, 3, 0, 6, 0, 4, 6, X, X, X, X, X, X, X}, + {8, 6, 11, 8, 4, 6, 9, 0, 1, X, X, X, X, X, X, X}, + {9, 4, 6, 9, 6, 3, 9, 3, 1, 11, 3, 6, X, X, X, X}, + {6, 8, 4, 6, 11, 8, 2, 10, 1, X, X, X, X, X, X, X}, + {1, 2, 10, 3, 0, 11, 0, 6, 11, 0, 4, 6, X, X, X, X}, + {4, 11, 8, 4, 6, 11, 0, 2, 9, 2, 10, 9, X, X, X, X}, + {10, 9, 3, 10, 3, 2, 9, 4, 3, 11, 3, 6, 4, 6, 3, X}, + {8, 2, 3, 8, 4, 2, 4, 6, 2, X, X, X, X, X, X, X}, + {0, 4, 2, 4, 6, 2, X, X, X, X, X, X, X, X, X, X}, + {1, 9, 0, 2, 3, 4, 2, 4, 6, 4, 3, 8, X, X, X, X}, + {1, 9, 4, 1, 4, 2, 2, 4, 6, X, X, X, X, X, X, X}, + {8, 1, 3, 8, 6, 1, 8, 4, 6, 6, 10, 1, X, X, X, X}, + {10, 1, 0, 10, 0, 6, 6, 0, 4, X, X, X, X, X, X, X}, + {4, 6, 3, 4, 3, 8, 6, 10, 3, 0, 3, 9, 10, 9, 3, X}, + {10, 9, 4, 6, 10, 4, X, X, X, X, X, X, X, X, X, X}, + {4, 9, 5, 7, 6, 11, X, X, X, X, X, X, X, X, X, X}, + {0, 8, 3, 4, 9, 5, 11, 7, 6, X, X, X, X, X, X, X}, + {5, 0, 1, 5, 4, 0, 7, 6, 11, X, X, X, X, X, X, X}, + {11, 7, 6, 8, 3, 4, 3, 5, 4, 3, 1, 5, X, X, X, X}, + {9, 5, 4, 10, 1, 2, 7, 6, 11, X, X, X, X, X, X, X}, + {6, 11, 7, 1, 2, 10, 0, 8, 3, 4, 9, 5, X, X, X, X}, + {7, 6, 11, 5, 4, 10, 4, 2, 10, 4, 0, 2, X, X, X, X}, + {3, 4, 8, 3, 5, 4, 3, 2, 5, 10, 5, 2, 11, 7, 6, X}, + {7, 2, 3, 7, 6, 2, 5, 4, 9, X, X, X, X, X, X, X}, + {9, 5, 4, 0, 8, 6, 0, 6, 2, 6, 8, 7, X, X, X, X}, + {3, 6, 2, 3, 7, 6, 1, 5, 0, 5, 4, 0, X, X, X, X}, + {6, 2, 8, 6, 8, 7, 2, 1, 8, 4, 8, 5, 1, 5, 8, X}, + {9, 5, 4, 10, 1, 6, 1, 7, 6, 1, 3, 7, X, X, X, X}, + {1, 6, 10, 1, 7, 6, 1, 0, 7, 8, 7, 0, 9, 5, 4, X}, + {4, 0, 10, 4, 10, 5, 0, 3, 10, 6, 10, 7, 3, 7, 10, X}, + {7, 6, 10, 7, 10, 8, 5, 4, 10, 4, 8, 10, X, X, X, X}, + {6, 9, 5, 6, 11, 9, 11, 8, 9, X, X, X, X, X, X, X}, + {3, 6, 11, 0, 6, 3, 0, 5, 6, 0, 9, 5, X, X, X, X}, + {0, 11, 8, 0, 5, 11, 0, 1, 5, 5, 6, 11, X, X, X, X}, + {6, 11, 3, 6, 3, 5, 5, 3, 1, X, X, X, X, X, X, X}, + {1, 2, 10, 9, 5, 11, 9, 11, 8, 11, 5, 6, X, X, X, X}, + {0, 11, 3, 0, 6, 11, 0, 9, 6, 5, 6, 9, 1, 2, 10, X}, + {11, 8, 5, 11, 5, 6, 8, 0, 5, 10, 5, 2, 0, 2, 5, X}, + {6, 11, 3, 6, 3, 5, 2, 10, 3, 10, 5, 3, X, X, X, X}, + {5, 8, 9, 5, 2, 8, 5, 6, 2, 3, 8, 2, X, X, X, X}, + {9, 5, 6, 9, 6, 0, 0, 6, 2, X, X, X, X, X, X, X}, + {1, 5, 8, 1, 8, 0, 5, 6, 8, 3, 8, 2, 6, 2, 8, X}, + {1, 5, 6, 2, 1, 6, X, X, X, X, X, X, X, X, X, X}, + {1, 3, 6, 1, 6, 10, 3, 8, 6, 5, 6, 9, 8, 9, 6, X}, + {10, 1, 0, 10, 0, 6, 9, 5, 0, 5, 6, 0, X, X, X, X}, + {0, 3, 8, 5, 6, 10, X, X, X, X, X, X, X, X, X, X}, + {10, 5, 6, X, X, X, X, X, X, X, X, X, X, X, X, X}, + {11, 5, 10, 7, 5, 11, X, X, X, X, X, X, X, X, X, X}, + {11, 5, 10, 11, 7, 5, 8, 3, 0, X, X, X, X, X, X, X}, + {5, 11, 7, 5, 10, 11, 1, 9, 0, X, X, X, X, X, X, X}, + {10, 7, 5, 10, 11, 7, 9, 8, 1, 8, 3, 1, X, X, X, X}, + {11, 1, 2, 11, 7, 1, 7, 5, 1, X, X, X, X, X, X, X}, + {0, 8, 3, 1, 2, 7, 1, 7, 5, 7, 2, 11, X, X, X, X}, + {9, 7, 5, 9, 2, 7, 9, 0, 2, 2, 11, 7, X, X, X, X}, + {7, 5, 2, 7, 2, 11, 5, 9, 2, 3, 2, 8, 9, 8, 2, X}, + {2, 5, 10, 2, 3, 5, 3, 7, 5, X, X, X, X, X, X, X}, + {8, 2, 0, 8, 5, 2, 8, 7, 5, 10, 2, 5, X, X, X, X}, + {9, 0, 1, 5, 10, 3, 5, 3, 7, 3, 10, 2, X, X, X, X}, + {9, 8, 2, 9, 2, 1, 8, 7, 2, 10, 2, 5, 7, 5, 2, X}, + {1, 3, 5, 3, 7, 5, X, X, X, X, X, X, X, X, X, X}, + {0, 8, 7, 0, 7, 1, 1, 7, 5, X, X, X, X, X, X, X}, + {9, 0, 3, 9, 3, 5, 5, 3, 7, X, X, X, X, X, X, X}, + {9, 8, 7, 5, 9, 7, X, X, X, X, X, X, X, X, X, X}, + {5, 8, 4, 5, 10, 8, 10, 11, 8, X, X, X, X, X, X, X}, + {5, 0, 4, 5, 11, 0, 5, 10, 11, 11, 3, 0, X, X, X, X}, + {0, 1, 9, 8, 4, 10, 8, 10, 11, 10, 4, 5, X, X, X, X}, + {10, 11, 4, 10, 4, 5, 11, 3, 4, 9, 4, 1, 3, 1, 4, X}, + {2, 5, 1, 2, 8, 5, 2, 11, 8, 4, 5, 8, X, X, X, X}, + {0, 4, 11, 0, 11, 3, 4, 5, 11, 2, 11, 1, 5, 1, 11, X}, + {0, 2, 5, 0, 5, 9, 2, 11, 5, 4, 5, 8, 11, 8, 5, X}, + {9, 4, 5, 2, 11, 3, X, X, X, X, X, X, X, X, X, X}, + {2, 5, 10, 3, 5, 2, 3, 4, 5, 3, 8, 4, X, X, X, X}, + {5, 10, 2, 5, 2, 4, 4, 2, 0, X, X, X, X, X, X, X}, + {3, 10, 2, 3, 5, 10, 3, 8, 5, 4, 5, 8, 0, 1, 9, X}, + {5, 10, 2, 5, 2, 4, 1, 9, 2, 9, 4, 2, X, X, X, X}, + {8, 4, 5, 8, 5, 3, 3, 5, 1, X, X, X, X, X, X, X}, + {0, 4, 5, 1, 0, 5, X, X, X, X, X, X, X, X, X, X}, + {8, 4, 5, 8, 5, 3, 9, 0, 5, 0, 3, 5, X, X, X, X}, + {9, 4, 5, X, X, X, X, X, X, X, X, X, X, X, X, X}, + {4, 11, 7, 4, 9, 11, 9, 10, 11, X, X, X, X, X, X, X}, + {0, 8, 3, 4, 9, 7, 9, 11, 7, 9, 10, 11, X, X, X, X}, + {1, 10, 11, 1, 11, 4, 1, 4, 0, 7, 4, 11, X, X, X, X}, + {3, 1, 4, 3, 4, 8, 1, 10, 4, 7, 4, 11, 10, 11, 4, X}, + {4, 11, 7, 9, 11, 4, 9, 2, 11, 9, 1, 2, X, X, X, X}, + {9, 7, 4, 9, 11, 7, 9, 1, 11, 2, 11, 1, 0, 8, 3, X}, + {11, 7, 4, 11, 4, 2, 2, 4, 0, X, X, X, X, X, X, X}, + {11, 7, 4, 11, 4, 2, 8, 3, 4, 3, 2, 4, X, X, X, X}, + {2, 9, 10, 2, 7, 9, 2, 3, 7, 7, 4, 9, X, X, X, X}, + {9, 10, 7, 9, 7, 4, 10, 2, 7, 8, 7, 0, 2, 0, 7, X}, + {3, 7, 10, 3, 10, 2, 7, 4, 10, 1, 10, 0, 4, 0, 10, X}, + {1, 10, 2, 8, 7, 4, X, X, X, X, X, X, X, X, X, X}, + {4, 9, 1, 4, 1, 7, 7, 1, 3, X, X, X, X, X, X, X}, + {4, 9, 1, 4, 1, 7, 0, 8, 1, 8, 7, 1, X, X, X, X}, + {4, 0, 3, 7, 4, 3, X, X, X, X, X, X, X, X, X, X}, + {4, 8, 7, X, X, X, X, X, X, X, X, X, X, X, X, X}, + {9, 10, 8, 10, 11, 8, X, X, X, X, X, X, X, X, X, X}, + {3, 0, 9, 3, 9, 11, 11, 9, 10, X, X, X, X, X, X, X}, + {0, 1, 10, 0, 10, 8, 8, 10, 11, X, X, X, X, X, X, X}, + {3, 1, 10, 11, 3, 10, X, X, X, X, X, X, X, X, X, X}, + {1, 2, 11, 1, 11, 9, 9, 11, 8, X, X, X, X, X, X, X}, + {3, 0, 9, 3, 9, 11, 1, 2, 9, 2, 11, 9, X, X, X, X}, + {0, 2, 11, 8, 0, 11, X, X, X, X, X, X, X, X, X, X}, + {3, 2, 11, X, X, X, X, X, X, X, X, X, X, X, X, X}, + {2, 3, 8, 2, 8, 10, 10, 8, 9, X, X, X, X, X, X, X}, + {9, 10, 2, 0, 9, 2, X, X, X, X, X, X, X, X, X, X}, + {2, 3, 8, 2, 8, 10, 0, 1, 8, 1, 10, 8, X, X, X, X}, + {1, 10, 2, X, X, X, X, X, X, X, X, X, X, X, X, X}, + {1, 3, 8, 9, 1, 8, X, X, X, X, X, X, X, X, X, X}, + {0, 9, 1, X, X, X, X, X, X, X, X, X, X, X, X, X}, + {0, 3, 8, X, X, X, X, X, X, X, X, X, X, X, X, X}, + {X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X}}; +#undef X + +// Table mapping each edge to the corresponding cube vertices offsets +const uint _EDGE_TO_VERTICES[12][2] = { + {0, 1}, + {1, 5}, + {4, 5}, + {0, 4}, + {2, 3}, + {3, 7}, + {6, 7}, + {2, 6}, + {0, 2}, + {1, 3}, + {5, 7}, + {4, 6}, +}; + +// Table mapping from 0-7 to v0-v7 in cube.vertices +const int _INDEX_TABLE[8] = {0, 1, 5, 4, 2, 3, 7, 6}; diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/mesh_normal_consistency/mesh_normal_consistency.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/mesh_normal_consistency/mesh_normal_consistency.h new file mode 100644 index 0000000000000000000000000000000000000000..17795ae1eb4b6bceb8a9960bc6d7523eb3c2acb6 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/mesh_normal_consistency/mesh_normal_consistency.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once +#include +#include "utils/pytorch3d_cutils.h" + +// For mesh_normal_consistency, find pairs of vertices opposite the same edge. +// +// Args: +// edge_num: int64 Tensor of shape (E,) giving the number of vertices +// corresponding to each edge. +// +// Returns: +// pairs: int64 Tensor of shape (N,2) + +at::Tensor MeshNormalConsistencyFindVerticesCpu(const at::Tensor& edge_num); + +// Exposed implementation. +at::Tensor MeshNormalConsistencyFindVertices(const at::Tensor& edge_num) { + if (edge_num.is_cuda()) { + AT_ERROR("This function needs a CPU tensor."); + } + return MeshNormalConsistencyFindVerticesCpu(edge_num); +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/mesh_normal_consistency/mesh_normal_consistency_cpu.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/mesh_normal_consistency/mesh_normal_consistency_cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1b0d5e6ea0a76cb2aac5e3f6ddb8eca77635da97 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/mesh_normal_consistency/mesh_normal_consistency_cpu.cpp @@ -0,0 +1,53 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include + +at::Tensor MeshNormalConsistencyFindVerticesCpu(const at::Tensor& edge_num) { + // We take a LongTensor of shape (E,) giving the number of things intersecting + // each edge. The things are taken to be numbered in order. + // (In fact, the "things" are opposite vertices to edges, renumbered). + // We return a tensor of shape (?, 2) where for every pair of things which + // intersect the same edge there is a row of their numbers in the output. + + // Example possible inputs and outputs (order of output is not specified): + // [1,0,1,1,0] => [[]] + // [3] => [[0,1], [0,2], [1,2]] + // [0,3] => [[0,1], [0,2], [1,2]] + // [1,3] => [[1,2], [1,3], [2,3]] + //[1,0,2,1,0,2] => [[1,2], [4,5]] + + const auto num_edges = edge_num.size(0); + auto edges_a = edge_num.accessor(); + + int64_t vert_idx = 0; + std::vector> pairs; + for (int64_t i_edge = 0; i_edge < num_edges; ++i_edge) { + int64_t e = edges_a[i_edge]; + for (int64_t j = 0; j < e; ++j) { + for (int64_t i = 0; i < j; ++i) { + pairs.emplace_back(vert_idx + i, vert_idx + j); + } + } + vert_idx += e; + } + + // Convert from std::vector by copying over the items to a new empty torch + // tensor. + auto pairs_tensor = at::empty({(int64_t)pairs.size(), 2}, edge_num.options()); + auto pairs_a = pairs_tensor.accessor(); + for (int64_t i_pair = 0; i_pair < pairs.size(); ++i_pair) { + auto accessor = pairs_a[i_pair]; + accessor[0] = pairs[i_pair].first; + accessor[1] = pairs[i_pair].second; + } + + return pairs_tensor; +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/packed_to_padded_tensor/packed_to_padded_tensor.cu b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/packed_to_padded_tensor/packed_to_padded_tensor.cu new file mode 100644 index 0000000000000000000000000000000000000000..94f22c18431bb8bc4557584acdd5894155a17e37 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/packed_to_padded_tensor/packed_to_padded_tensor.cu @@ -0,0 +1,241 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include + +// Kernel for inputs_packed of shape (F, D), where D > 1 +template +__global__ void PackedToPaddedKernel( + const scalar_t* __restrict__ inputs_packed, + const int64_t* __restrict__ first_idxs, + scalar_t* __restrict__ inputs_padded, + const size_t batch_size, + const size_t max_size, + const size_t num_inputs, + const size_t D) { + // Batch elements split evenly across blocks (num blocks = batch_size) and + // values for each element split across threads in the block. Each thread adds + // the values of its respective input elements to the global inputs_padded + // tensor. + const size_t tid = threadIdx.x; + const size_t batch_idx = blockIdx.x; + + const int64_t start = first_idxs[batch_idx]; + const int64_t end = + batch_idx + 1 < batch_size ? first_idxs[batch_idx + 1] : num_inputs; + const int num = end - start; + for (size_t f = tid; f < num; f += blockDim.x) { + for (size_t j = 0; j < D; ++j) { + inputs_padded[batch_idx * max_size * D + f * D + j] = + inputs_packed[(start + f) * D + j]; + } + } +} + +// Kernel for inputs of shape (F, 1) +template +__global__ void PackedToPaddedKernelD1( + const scalar_t* __restrict__ inputs_packed, + const int64_t* __restrict__ first_idxs, + scalar_t* __restrict__ inputs_padded, + const size_t batch_size, + const size_t max_size, + const size_t num_inputs) { + // Batch elements split evenly across blocks (num blocks = batch_size) and + // values for each element split across threads in the block. Each thread adds + // the values of its respective input elements to the global inputs_padded + // tensor. + const size_t tid = threadIdx.x; + const size_t batch_idx = blockIdx.x; + + const int64_t start = first_idxs[batch_idx]; + const int64_t end = + batch_idx + 1 < batch_size ? first_idxs[batch_idx + 1] : num_inputs; + const int num = end - start; + for (size_t f = tid; f < num; f += blockDim.x) { + inputs_padded[batch_idx * max_size + f] = inputs_packed[start + f]; + } +} + +// Kernel for inputs_padded of shape (B, F, D), where D > 1 +template +__global__ void PaddedToPackedKernel( + const scalar_t* __restrict__ inputs_padded, + const int64_t* __restrict__ first_idxs, + scalar_t* __restrict__ inputs_packed, + const size_t batch_size, + const size_t max_size, + const size_t num_inputs, + const size_t D) { + // Batch elements split evenly across blocks (num blocks = batch_size) and + // values for each element split across threads in the block. Each thread adds + // the values of its respective input elements to the global inputs_packed + // tensor. + const size_t tid = threadIdx.x; + const size_t batch_idx = blockIdx.x; + + const int64_t start = first_idxs[batch_idx]; + const int64_t end = + batch_idx + 1 < batch_size ? first_idxs[batch_idx + 1] : num_inputs; + const int num = end - start; + for (size_t f = tid; f < num; f += blockDim.x) { + for (size_t j = 0; j < D; ++j) { + inputs_packed[(start + f) * D + j] = + inputs_padded[batch_idx * max_size * D + f * D + j]; + } + } +} + +// Kernel for inputs_padded of shape (B, F, 1) +template +__global__ void PaddedToPackedKernelD1( + const scalar_t* __restrict__ inputs_padded, + const int64_t* __restrict__ first_idxs, + scalar_t* __restrict__ inputs_packed, + const size_t batch_size, + const size_t max_size, + const size_t num_inputs) { + // Batch elements split evenly across blocks (num blocks = batch_size) and + // values for each element split across threads in the block. Each thread adds + // the values of its respective input elements to the global inputs_packed + // tensor. + const size_t tid = threadIdx.x; + const size_t batch_idx = blockIdx.x; + + const int64_t start = first_idxs[batch_idx]; + const int64_t end = + batch_idx + 1 < batch_size ? first_idxs[batch_idx + 1] : num_inputs; + const int num = end - start; + for (size_t f = tid; f < num; f += blockDim.x) { + inputs_packed[start + f] = inputs_padded[batch_idx * max_size + f]; + } +} + +at::Tensor PackedToPaddedCuda( + const at::Tensor inputs_packed, + const at::Tensor first_idxs, + const int64_t max_size) { + // Check inputs are on the same device + at::TensorArg inputs_packed_t{inputs_packed, "inputs_packed", 1}, + first_idxs_t{first_idxs, "first_idxs", 2}; + at::CheckedFrom c = "PackedToPaddedCuda"; + at::checkAllSameGPU(c, {inputs_packed_t, first_idxs_t}); + + // Set the device for the kernel launch based on the device of the input + at::cuda::CUDAGuard device_guard(inputs_packed.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + const int64_t num_inputs = inputs_packed.size(0); + const int64_t batch_size = first_idxs.size(0); + + TORCH_CHECK( + inputs_packed.dim() == 2, "inputs_packed must be a 2-dimensional tensor"); + const int64_t D = inputs_packed.size(1); + at::Tensor inputs_padded = + at::zeros({batch_size, max_size, D}, inputs_packed.options()); + + if (inputs_padded.numel() == 0) { + AT_CUDA_CHECK(cudaGetLastError()); + return inputs_padded; + } + + const int threads = 512; + const int blocks = batch_size; + if (D == 1) { + AT_DISPATCH_FLOATING_TYPES( + inputs_packed.scalar_type(), "packed_to_padded_d1_kernel", ([&] { + PackedToPaddedKernelD1<<>>( + inputs_packed.contiguous().data_ptr(), + first_idxs.contiguous().data_ptr(), + inputs_padded.data_ptr(), + batch_size, + max_size, + num_inputs); + })); + } else { + AT_DISPATCH_FLOATING_TYPES( + inputs_packed.scalar_type(), "packed_to_padded_kernel", ([&] { + PackedToPaddedKernel<<>>( + inputs_packed.contiguous().data_ptr(), + first_idxs.contiguous().data_ptr(), + inputs_padded.data_ptr(), + batch_size, + max_size, + num_inputs, + D); + })); + } + + AT_CUDA_CHECK(cudaGetLastError()); + return inputs_padded; +} + +at::Tensor PaddedToPackedCuda( + const at::Tensor inputs_padded, + const at::Tensor first_idxs, + const int64_t num_inputs) { + // Check inputs are on the same device + at::TensorArg inputs_padded_t{inputs_padded, "inputs_padded", 1}, + first_idxs_t{first_idxs, "first_idxs", 2}; + at::CheckedFrom c = "PaddedToPackedCuda"; + at::checkAllSameGPU(c, {inputs_padded_t, first_idxs_t}); + + // Set the device for the kernel launch based on the device of the input + at::cuda::CUDAGuard device_guard(inputs_padded.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + const int64_t batch_size = inputs_padded.size(0); + const int64_t max_size = inputs_padded.size(1); + + TORCH_CHECK(batch_size == first_idxs.size(0), "sizes mismatch"); + TORCH_CHECK( + inputs_padded.dim() == 3, + "inputs_padded must be a 3-dimensional tensor"); + const int64_t D = inputs_padded.size(2); + + at::Tensor inputs_packed = + at::zeros({num_inputs, D}, inputs_padded.options()); + + if (inputs_packed.numel() == 0) { + AT_CUDA_CHECK(cudaGetLastError()); + return inputs_packed; + } + + const int threads = 512; + const int blocks = batch_size; + + if (D == 1) { + AT_DISPATCH_FLOATING_TYPES( + inputs_padded.scalar_type(), "padded_to_packed_d1_kernel", ([&] { + PaddedToPackedKernelD1<<>>( + inputs_padded.contiguous().data_ptr(), + first_idxs.contiguous().data_ptr(), + inputs_packed.data_ptr(), + batch_size, + max_size, + num_inputs); + })); + } else { + AT_DISPATCH_FLOATING_TYPES( + inputs_padded.scalar_type(), "padded_to_packed_kernel", ([&] { + PaddedToPackedKernel<<>>( + inputs_padded.contiguous().data_ptr(), + first_idxs.contiguous().data_ptr(), + inputs_packed.data_ptr(), + batch_size, + max_size, + num_inputs, + D); + })); + } + + AT_CUDA_CHECK(cudaGetLastError()); + return inputs_packed; +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/packed_to_padded_tensor/packed_to_padded_tensor.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/packed_to_padded_tensor/packed_to_padded_tensor.h new file mode 100644 index 0000000000000000000000000000000000000000..27bbc3bf03d1e195f44d36662c93d9db85adae68 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/packed_to_padded_tensor/packed_to_padded_tensor.h @@ -0,0 +1,109 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once +#include +#include "utils/pytorch3d_cutils.h" + +// PackedToPadded +// Converts a packed tensor into a padded tensor, restoring the batch dimension. +// Refer to pytorch3d/structures/meshes.py for details on packed/padded tensors. +// +// Inputs: +// inputs_packed: FloatTensor of shape (F, D), representing the packed batch +// tensor, e.g. areas for faces in a batch of meshes. +// first_idxs: LongTensor of shape (N,) where N is the number of +// elements in the batch and `first_idxs[i] = f` +// means that the inputs for batch element i begin at +// `inputs[f]`. +// max_size: Max length of an element in the batch. +// Returns: +// inputs_padded: FloatTensor of shape (N, max_size, D) where max_size is max +// of `sizes`. The values for batch element i which start at +// `inputs_packed[first_idxs[i]]` will be copied to +// `inputs_padded[i, :]`, with zeros padding out the extra +// inputs. +// + +// PaddedToPacked +// Converts a padded tensor into a packed tensor. +// Refer to pytorch3d/structures/meshes.py for details on packed/padded tensors. +// +// Inputs: +// inputs_padded: FloatTensor of shape (N, max_size, D), representing the +// padded tensor, e.g. areas for faces in a batch of meshes. +// first_idxs: LongTensor of shape (N,) where N is the number of +// elements in the batch and `first_idxs[i] = f` +// means that the inputs for batch element i begin at +// `inputs_packed[f]`. +// num_inputs: Number of packed entries (= F) +// Returns: +// inputs_packed: FloatTensor of shape (F, D), where +// `inputs_packed[first_idx[i]:] = inputs_padded[i, :]`. +// +// + +// Cpu implementation. +at::Tensor PackedToPaddedCpu( + const at::Tensor inputs_packed, + const at::Tensor first_idxs, + const int64_t max_size); + +// Cpu implementation. +at::Tensor PaddedToPackedCpu( + const at::Tensor inputs_padded, + const at::Tensor first_idxs, + const int64_t num_inputs); + +#ifdef WITH_CUDA +// Cuda implementation. +at::Tensor PackedToPaddedCuda( + const at::Tensor inputs_packed, + const at::Tensor first_idxs, + const int64_t max_size); + +// Cuda implementation. +at::Tensor PaddedToPackedCuda( + const at::Tensor inputs_padded, + const at::Tensor first_idxs, + const int64_t num_inputs); +#endif + +// Implementation which is exposed. +at::Tensor PackedToPadded( + const at::Tensor inputs_packed, + const at::Tensor first_idxs, + const int64_t max_size) { + if (inputs_packed.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(inputs_packed); + CHECK_CUDA(first_idxs); + return PackedToPaddedCuda(inputs_packed, first_idxs, max_size); +#else + AT_ERROR("Not compiled with GPU support."); +#endif + } + return PackedToPaddedCpu(inputs_packed, first_idxs, max_size); +} + +// Implementation which is exposed. +at::Tensor PaddedToPacked( + const at::Tensor inputs_padded, + const at::Tensor first_idxs, + const int64_t num_inputs) { + if (inputs_padded.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(inputs_padded); + CHECK_CUDA(first_idxs); + return PaddedToPackedCuda(inputs_padded, first_idxs, num_inputs); +#else + AT_ERROR("Not compiled with GPU support."); +#endif + } + return PaddedToPackedCpu(inputs_padded, first_idxs, num_inputs); +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/packed_to_padded_tensor/packed_to_padded_tensor_cpu.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/packed_to_padded_tensor/packed_to_padded_tensor_cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c9176a1afd5e6736f938dc938dfc1d62c1052ddc --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/packed_to_padded_tensor/packed_to_padded_tensor_cpu.cpp @@ -0,0 +1,70 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +at::Tensor PackedToPaddedCpu( + const at::Tensor inputs_packed, + const at::Tensor first_idxs, + const int64_t max_size) { + const int64_t num_inputs = inputs_packed.size(0); + const int64_t batch_size = first_idxs.size(0); + + AT_ASSERTM( + inputs_packed.dim() == 2, "inputs_packed must be a 2-dimensional tensor"); + const int64_t D = inputs_packed.size(1); + + torch::Tensor inputs_padded = + torch::zeros({batch_size, max_size, D}, inputs_packed.options()); + + auto inputs_packed_a = inputs_packed.accessor(); + auto first_idxs_a = first_idxs.accessor(); + auto inputs_padded_a = inputs_padded.accessor(); + + for (int b = 0; b < batch_size; ++b) { + const int64_t start = first_idxs_a[b]; + const int64_t end = b + 1 < batch_size ? first_idxs_a[b + 1] : num_inputs; + const int64_t num = end - start; + for (int i = 0; i < num; ++i) { + for (int j = 0; j < D; ++j) { + inputs_padded_a[b][i][j] = inputs_packed_a[start + i][j]; + } + } + } + return inputs_padded; +} + +at::Tensor PaddedToPackedCpu( + const at::Tensor inputs_padded, + const at::Tensor first_idxs, + const int64_t num_inputs) { + const int64_t batch_size = inputs_padded.size(0); + + AT_ASSERTM( + inputs_padded.dim() == 3, "inputs_padded must be a 3-dimensional tensor"); + const int64_t D = inputs_padded.size(2); + + torch::Tensor inputs_packed = + torch::zeros({num_inputs, D}, inputs_padded.options()); + + auto inputs_padded_a = inputs_padded.accessor(); + auto first_idxs_a = first_idxs.accessor(); + auto inputs_packed_a = inputs_packed.accessor(); + + for (int b = 0; b < batch_size; ++b) { + const int64_t start = first_idxs_a[b]; + const int64_t end = b + 1 < batch_size ? first_idxs_a[b + 1] : num_inputs; + const int64_t num = end - start; + for (int i = 0; i < num; ++i) { + for (int j = 0; j < D; ++j) { + inputs_packed_a[start + i][j] = inputs_padded_a[b][i][j]; + } + } + } + return inputs_packed; +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/point_mesh/point_mesh_cpu.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/point_mesh/point_mesh_cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e059409c40330005e76f88792bf7037cb72e4000 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/point_mesh/point_mesh_cpu.cpp @@ -0,0 +1,428 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include +#include "utils/geometry_utils.h" +#include "utils/vec3.h" + +// - We start with implementations of simple operations on points, edges and +// faces. The hull of H points is a point if H=1, an edge if H=2, a face if H=3. + +template +vec3 ExtractPoint(const at::TensorAccessor& t) { + return vec3(t[0], t[1], t[2]); +} + +template +static std::array>, 1> +ExtractHullHelper(const Accessor& t, std::array /*tag*/) { + return {ExtractPoint(t)}; +} + +template +static std::array>, 2> +ExtractHullHelper(const Accessor& t, std::array /*tag*/) { + return {ExtractPoint(t[0]), ExtractPoint(t[1])}; +} + +template +static std::array>, 3> +ExtractHullHelper(const Accessor& t, std::array /*tag*/) { + return {ExtractPoint(t[0]), ExtractPoint(t[1]), ExtractPoint(t[2])}; +} + +template +std::array>, H> +ExtractHull(const Accessor& t) { + std::array tag; + return ExtractHullHelper(t, tag); +} + +template +void IncrementPoint(at::TensorAccessor&& t, const vec3& point) { + t[0] += point.x; + t[1] += point.y; + t[2] += point.z; +} + +// distance between the convex hull of A points and B points +// this could be done in c++17 with tuple_cat and invoke +template +T HullDistance( + const std::array, 1>& a, + const std::array, 2>& b, + const double /*min_triangle_area*/) { + using std::get; + return PointLine3DistanceForward(get<0>(a), get<0>(b), get<1>(b)); +} +template +T HullDistance( + const std::array, 1>& a, + const std::array, 3>& b, + const double min_triangle_area) { + using std::get; + return PointTriangle3DistanceForward( + get<0>(a), get<0>(b), get<1>(b), get<2>(b), min_triangle_area); +} +template +T HullDistance( + const std::array, 2>& a, + const std::array, 1>& b, + const double /*min_triangle_area*/) { + return HullDistance(b, a, 1); +} +template +T HullDistance( + const std::array, 3>& a, + const std::array, 1>& b, + const double min_triangle_area) { + return HullDistance(b, a, min_triangle_area); +} + +template +void HullHullDistanceBackward( + const std::array, 1>& a, + const std::array, 2>& b, + T grad_dist, + at::TensorAccessor&& grad_a, + at::TensorAccessor&& grad_b, + const double /*min_triangle_area*/) { + using std::get; + auto res = + PointLine3DistanceBackward(get<0>(a), get<0>(b), get<1>(b), grad_dist); + IncrementPoint(std::move(grad_a), get<0>(res)); + IncrementPoint(grad_b[0], get<1>(res)); + IncrementPoint(grad_b[1], get<2>(res)); +} +template +void HullHullDistanceBackward( + const std::array, 1>& a, + const std::array, 3>& b, + T grad_dist, + at::TensorAccessor&& grad_a, + at::TensorAccessor&& grad_b, + const double min_triangle_area) { + using std::get; + auto res = PointTriangle3DistanceBackward( + get<0>(a), get<0>(b), get<1>(b), get<2>(b), grad_dist, min_triangle_area); + IncrementPoint(std::move(grad_a), get<0>(res)); + IncrementPoint(grad_b[0], get<1>(res)); + IncrementPoint(grad_b[1], get<2>(res)); + IncrementPoint(grad_b[2], get<3>(res)); +} +template +void HullHullDistanceBackward( + const std::array, 3>& a, + const std::array, 1>& b, + T grad_dist, + at::TensorAccessor&& grad_a, + at::TensorAccessor&& grad_b, + const double min_triangle_area) { + return HullHullDistanceBackward( + b, a, grad_dist, std::move(grad_b), std::move(grad_a), min_triangle_area); +} +template +void HullHullDistanceBackward( + const std::array, 2>& a, + const std::array, 1>& b, + T grad_dist, + at::TensorAccessor&& grad_a, + at::TensorAccessor&& grad_b, + const double /*min_triangle_area*/) { + return HullHullDistanceBackward( + b, a, grad_dist, std::move(grad_b), std::move(grad_a), 1); +} + +template +void ValidateShape(const at::Tensor& as) { + if (H == 1) { + TORCH_CHECK(as.size(1) == 3); + } else { + TORCH_CHECK(as.size(2) == 3 && as.size(1) == H); + } +} + +// ----------- Here begins the implementation of each top-level +// function using non-type template parameters to +// implement all the cases in one go. ----------- // + +template +std::tuple HullHullDistanceForwardCpu( + const at::Tensor& as, + const at::Tensor& as_first_idx, + const at::Tensor& bs, + const at::Tensor& bs_first_idx, + const double min_triangle_area) { + const int64_t A_N = as.size(0); + const int64_t B_N = bs.size(0); + const int64_t BATCHES = as_first_idx.size(0); + + ValidateShape

(as); + ValidateShape

(bs); + + TORCH_CHECK(bs_first_idx.size(0) == BATCHES); + + // clang-format off + at::Tensor dists = at::zeros({A_N,}, as.options()); + at::Tensor idxs = at::zeros({A_N,}, as_first_idx.options()); + // clang-format on + + auto as_a = as.accessor < float, H1 == 1 ? 2 : 3 > (); + auto bs_a = bs.accessor < float, H2 == 1 ? 2 : 3 > (); + auto as_first_idx_a = as_first_idx.accessor(); + auto bs_first_idx_a = bs_first_idx.accessor(); + auto dists_a = dists.accessor(); + auto idxs_a = idxs.accessor(); + int64_t a_batch_end = 0; + int64_t b_batch_start = 0, b_batch_end = 0; + int64_t batch_idx = 0; + for (int64_t a_n = 0; a_n < A_N; ++a_n) { + if (a_n == a_batch_end) { + ++batch_idx; + b_batch_start = b_batch_end; + if (batch_idx == BATCHES) { + a_batch_end = std::numeric_limits::max(); + b_batch_end = B_N; + } else { + a_batch_end = as_first_idx_a[batch_idx]; + b_batch_end = bs_first_idx_a[batch_idx]; + } + } + float min_dist = std::numeric_limits::max(); + size_t min_idx = 0; + auto a = ExtractHull

(as_a[a_n]); + for (int64_t b_n = b_batch_start; b_n < b_batch_end; ++b_n) { + float dist = + HullDistance(a, ExtractHull

(bs_a[b_n]), min_triangle_area); + if (dist <= min_dist) { + min_dist = dist; + min_idx = b_n; + } + } + dists_a[a_n] = min_dist; + idxs_a[a_n] = min_idx; + } + + return std::make_tuple(dists, idxs); +} + +template +std::tuple HullHullDistanceBackwardCpu( + const at::Tensor& as, + const at::Tensor& bs, + const at::Tensor& idx_bs, + const at::Tensor& grad_dists, + const double min_triangle_area) { + const int64_t A_N = as.size(0); + + TORCH_CHECK(idx_bs.size(0) == A_N); + TORCH_CHECK(grad_dists.size(0) == A_N); + ValidateShape

(as); + ValidateShape

(bs); + + at::Tensor grad_as = at::zeros_like(as); + at::Tensor grad_bs = at::zeros_like(bs); + + auto as_a = as.accessor < float, H1 == 1 ? 2 : 3 > (); + auto bs_a = bs.accessor < float, H2 == 1 ? 2 : 3 > (); + auto grad_as_a = grad_as.accessor < float, H1 == 1 ? 2 : 3 > (); + auto grad_bs_a = grad_bs.accessor < float, H2 == 1 ? 2 : 3 > (); + auto idx_bs_a = idx_bs.accessor(); + auto grad_dists_a = grad_dists.accessor(); + + for (int64_t a_n = 0; a_n < A_N; ++a_n) { + auto a = ExtractHull

(as_a[a_n]); + auto b = ExtractHull

(bs_a[idx_bs_a[a_n]]); + HullHullDistanceBackward( + a, + b, + grad_dists_a[a_n], + grad_as_a[a_n], + grad_bs_a[idx_bs_a[a_n]], + min_triangle_area); + } + return std::make_tuple(grad_as, grad_bs); +} + +template +torch::Tensor PointHullArrayDistanceForwardCpu( + const torch::Tensor& points, + const torch::Tensor& bs, + const double min_triangle_area) { + const int64_t P = points.size(0); + const int64_t B_N = bs.size(0); + + TORCH_CHECK(points.size(1) == 3, "points must be of shape Px3"); + ValidateShape(bs); + + at::Tensor dists = at::zeros({P, B_N}, points.options()); + auto points_a = points.accessor(); + auto bs_a = bs.accessor(); + auto dists_a = dists.accessor(); + for (int64_t p = 0; p < P; ++p) { + auto point = ExtractHull<1>(points_a[p]); + auto dest = dists_a[p]; + for (int64_t b_n = 0; b_n < B_N; ++b_n) { + auto b = ExtractHull(bs_a[b_n]); + dest[b_n] = HullDistance(point, b, min_triangle_area); + } + } + return dists; +} + +template +std::tuple PointHullArrayDistanceBackwardCpu( + const at::Tensor& points, + const at::Tensor& bs, + const at::Tensor& grad_dists, + const double min_triangle_area) { + const int64_t P = points.size(0); + const int64_t B_N = bs.size(0); + + TORCH_CHECK(points.size(1) == 3, "points must be of shape Px3"); + ValidateShape(bs); + TORCH_CHECK((grad_dists.size(0) == P) && (grad_dists.size(1) == B_N)); + + at::Tensor grad_points = at::zeros({P, 3}, points.options()); + at::Tensor grad_bs = at::zeros({B_N, H, 3}, bs.options()); + + auto points_a = points.accessor(); + auto bs_a = bs.accessor(); + auto grad_dists_a = grad_dists.accessor(); + auto grad_points_a = grad_points.accessor(); + auto grad_bs_a = grad_bs.accessor(); + for (int64_t p = 0; p < P; ++p) { + auto point = ExtractHull<1>(points_a[p]); + auto grad_point = grad_points_a[p]; + auto grad_dist = grad_dists_a[p]; + for (int64_t b_n = 0; b_n < B_N; ++b_n) { + auto b = ExtractHull(bs_a[b_n]); + HullHullDistanceBackward( + point, + b, + grad_dist[b_n], + std::move(grad_point), + grad_bs_a[b_n], + min_triangle_area); + } + } + return std::make_tuple(grad_points, grad_bs); +} + +// ---------- Here begin the exported functions ------------ // + +std::tuple PointFaceDistanceForwardCpu( + const torch::Tensor& points, + const torch::Tensor& points_first_idx, + const torch::Tensor& tris, + const torch::Tensor& tris_first_idx, + const double min_triangle_area) { + return HullHullDistanceForwardCpu<1, 3>( + points, points_first_idx, tris, tris_first_idx, min_triangle_area); +} + +std::tuple PointFaceDistanceBackwardCpu( + const torch::Tensor& points, + const torch::Tensor& tris, + const torch::Tensor& idx_points, + const torch::Tensor& grad_dists, + const double min_triangle_area) { + return HullHullDistanceBackwardCpu<1, 3>( + points, tris, idx_points, grad_dists, min_triangle_area); +} + +std::tuple FacePointDistanceForwardCpu( + const torch::Tensor& points, + const torch::Tensor& points_first_idx, + const torch::Tensor& tris, + const torch::Tensor& tris_first_idx, + const double min_triangle_area) { + return HullHullDistanceForwardCpu<3, 1>( + tris, tris_first_idx, points, points_first_idx, min_triangle_area); +} + +std::tuple FacePointDistanceBackwardCpu( + const torch::Tensor& points, + const torch::Tensor& tris, + const torch::Tensor& idx_tris, + const torch::Tensor& grad_dists, + const double min_triangle_area) { + auto res = HullHullDistanceBackwardCpu<3, 1>( + tris, points, idx_tris, grad_dists, min_triangle_area); + return std::make_tuple(std::get<1>(res), std::get<0>(res)); +} + +torch::Tensor PointEdgeArrayDistanceForwardCpu( + const torch::Tensor& points, + const torch::Tensor& segms) { + return PointHullArrayDistanceForwardCpu<2>(points, segms, 1); +} + +std::tuple PointFaceArrayDistanceBackwardCpu( + const at::Tensor& points, + const at::Tensor& tris, + const at::Tensor& grad_dists, + const double min_triangle_area) { + return PointHullArrayDistanceBackwardCpu<3>( + points, tris, grad_dists, min_triangle_area); +} + +torch::Tensor PointFaceArrayDistanceForwardCpu( + const torch::Tensor& points, + const torch::Tensor& tris, + const double min_triangle_area) { + return PointHullArrayDistanceForwardCpu<3>(points, tris, min_triangle_area); +} + +std::tuple PointEdgeArrayDistanceBackwardCpu( + const at::Tensor& points, + const at::Tensor& segms, + const at::Tensor& grad_dists) { + return PointHullArrayDistanceBackwardCpu<2>(points, segms, grad_dists, 1); +} + +std::tuple PointEdgeDistanceForwardCpu( + const torch::Tensor& points, + const torch::Tensor& points_first_idx, + const torch::Tensor& segms, + const torch::Tensor& segms_first_idx, + const int64_t /*max_points*/) { + return HullHullDistanceForwardCpu<1, 2>( + points, points_first_idx, segms, segms_first_idx, 1); +} + +std::tuple PointEdgeDistanceBackwardCpu( + const torch::Tensor& points, + const torch::Tensor& segms, + const torch::Tensor& idx_points, + const torch::Tensor& grad_dists) { + return HullHullDistanceBackwardCpu<1, 2>( + points, segms, idx_points, grad_dists, 1); +} + +std::tuple EdgePointDistanceForwardCpu( + const torch::Tensor& points, + const torch::Tensor& points_first_idx, + const torch::Tensor& segms, + const torch::Tensor& segms_first_idx, + const int64_t /*max_segms*/) { + return HullHullDistanceForwardCpu<2, 1>( + segms, segms_first_idx, points, points_first_idx, 1); +} + +std::tuple EdgePointDistanceBackwardCpu( + const torch::Tensor& points, + const torch::Tensor& segms, + const torch::Tensor& idx_segms, + const torch::Tensor& grad_dists) { + auto res = HullHullDistanceBackwardCpu<2, 1>( + segms, points, idx_segms, grad_dists, 1); + return std::make_tuple(std::get<1>(res), std::get<0>(res)); +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/point_mesh/point_mesh_cuda.cu b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/point_mesh/point_mesh_cuda.cu new file mode 100644 index 0000000000000000000000000000000000000000..3788d4055136e854f9415fef4e1233cfe23cfc86 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/point_mesh/point_mesh_cuda.cu @@ -0,0 +1,833 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "utils/float_math.cuh" +#include "utils/geometry_utils.cuh" +#include "utils/warp_reduce.cuh" + +// **************************************************************************** +// * Generic Forward/Backward Kernels * +// **************************************************************************** + +__global__ void DistanceForwardKernel( + const float* __restrict__ objects, // (O * oD * 3) + const size_t objects_size, // O + const size_t objects_dim, // oD + const float* __restrict__ targets, // (T * tD * 3) + const size_t targets_size, // T + const size_t targets_dim, // tD + const int64_t* __restrict__ objects_first_idx, // (B,) + const int64_t* __restrict__ targets_first_idx, // (B,) + const size_t batch_size, // B + float* __restrict__ dist_objects, // (O,) + int64_t* __restrict__ idx_objects, // (O,) + const double min_triangle_area) { + // This kernel is used interchangeably to compute bi-directional distances + // between points and triangles/lines. The direction of the distance computed, + // i.e. point to triangle/line or triangle/line to point, depends on the order + // of the input arguments and is inferred based on their shape. The shape is + // used to distinguish between triangles and lines + + // Single shared memory buffer which is split and cast to different types. + extern __shared__ char shared_buf[]; + float* min_dists = (float*)shared_buf; // float[NUM_THREADS] + int64_t* min_idxs = (int64_t*)&min_dists[blockDim.x]; // int64_t[NUM_THREADS] + + const size_t batch_idx = blockIdx.y; // index of batch element. + + // start and end for objects in batch_idx + const int64_t starto = objects_first_idx[batch_idx]; + const int64_t endo = batch_idx + 1 < batch_size + ? objects_first_idx[batch_idx + 1] + : objects_size; + + // start and end for targets in batch_idx + const int64_t startt = targets_first_idx[batch_idx]; + const int64_t endt = batch_idx + 1 < batch_size + ? targets_first_idx[batch_idx + 1] + : targets_size; + + const size_t i = blockIdx.x; // index within batch element. + const size_t tid = threadIdx.x; // thread index + + // Set references to points/face based on which of objects/targets refer to + // points/faces + float3* points_f3 = objects_dim == 1 ? (float3*)objects : (float3*)targets; + float3* face_f3 = objects_dim == 1 ? (float3*)targets : (float3*)objects; + // Distinguishes whether we're computing distance against triangle vs edge + bool isTriangle = objects_dim == 3 || targets_dim == 3; + + // Each block will compute one element of the output idx_objects[starto + i], + // dist_objects[starto + i]. Within the block we will use threads to compute + // the distances between objects[starto + i] and targets[j] for all j + // belonging in the same batch as i, i.e. j in [startt, endt]. Then use a + // block reduction to take an argmin of the distances. + + // If i exceeds the number of objects in batch_idx, then do nothing + if (i < (endo - starto)) { + // Compute the distances between objects[starto + i] and targets[j] for + // all j belonging in the same batch as i, i.e. j in [startt, endt]. + // Here each thread will reduce over (endt-startt) / blockDim.x in serial, + // and store its result to shared memory + float min_dist = FLT_MAX; + size_t min_idx = 0; + for (size_t j = tid; j < (endt - startt); j += blockDim.x) { + size_t point_idx = objects_dim == 1 ? starto + i : startt + j; + size_t face_idx = objects_dim == 1 ? (startt + j) * targets_dim + : (starto + i) * objects_dim; + + float dist; + if (isTriangle) { + dist = PointTriangle3DistanceForward( + points_f3[point_idx], + face_f3[face_idx], + face_f3[face_idx + 1], + face_f3[face_idx + 2], + min_triangle_area); + } else { + dist = PointLine3DistanceForward( + points_f3[point_idx], face_f3[face_idx], face_f3[face_idx + 1]); + } + + min_dist = (j == tid) ? dist : min_dist; + min_idx = (dist <= min_dist) ? (startt + j) : min_idx; + min_dist = (dist <= min_dist) ? dist : min_dist; + } + min_dists[tid] = min_dist; + min_idxs[tid] = min_idx; + __syncthreads(); + + // Perform reduction in shared memory. + for (int s = blockDim.x / 2; s > 32; s >>= 1) { + if (tid < s) { + if (min_dists[tid] > min_dists[tid + s]) { + min_dists[tid] = min_dists[tid + s]; + min_idxs[tid] = min_idxs[tid + s]; + } + } + __syncthreads(); + } + + // Unroll the last 6 iterations of the loop since they will happen + // synchronized within a single warp. + if (tid < 32) + WarpReduceMin(min_dists, min_idxs, tid); + + // Finally thread 0 writes the result to the output buffer. + if (tid == 0) { + idx_objects[starto + i] = min_idxs[0]; + dist_objects[starto + i] = min_dists[0]; + } + } +} + +std::tuple DistanceForwardCuda( + const at::Tensor& objects, + const size_t objects_dim, + const at::Tensor& objects_first_idx, + const at::Tensor& targets, + const size_t targets_dim, + const at::Tensor& targets_first_idx, + const int64_t max_objects, + const double min_triangle_area) { + // Check inputs are on the same device + at::TensorArg objects_t{objects, "objects", 1}, + objects_first_idx_t{objects_first_idx, "objects_first_idx", 2}, + targets_t{targets, "targets", 3}, + targets_first_idx_t{targets_first_idx, "targets_first_idx", 4}; + at::CheckedFrom c = "DistanceForwardCuda"; + at::checkAllSameGPU( + c, {objects_t, objects_first_idx_t, targets_t, targets_first_idx_t}); + at::checkAllSameType(c, {objects_t, targets_t}); + + // Set the device for the kernel launch based on the device of the input + at::cuda::CUDAGuard device_guard(objects.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + const int64_t objects_size = objects.size(0); + const int64_t targets_size = targets.size(0); + const int64_t batch_size = objects_first_idx.size(0); + + TORCH_CHECK(targets_first_idx.size(0) == batch_size); + if (objects_dim == 1) { + TORCH_CHECK( + targets_dim >= 2 && targets_dim <= 3, + "either object or target must be edge or face"); + TORCH_CHECK(objects.size(1) == 3, "points must be of shape Px3"); + TORCH_CHECK( + targets.size(2) == 3, + "face must be of shape Tx3x3, lines must be of shape Tx2x3"); + } else { + TORCH_CHECK(targets_dim == 1, "either object or target must be point"); + TORCH_CHECK( + objects_dim >= 2 && objects_dim <= 3, + "either object or target must be edge or face"); + TORCH_CHECK(targets.size(1) == 3, "points must be of shape Px3"); + TORCH_CHECK( + objects.size(2) == 3, + "face must be of shape Tx3x3, lines must be of shape Tx2x3"); + } + + // clang-format off + at::Tensor dists = at::zeros({objects_size,}, objects.options()); + at::Tensor idxs = at::zeros({objects_size,}, objects_first_idx.options()); + // clang-format on + + if (dists.numel() == 0) { + AT_CUDA_CHECK(cudaGetLastError()); + return std::make_tuple(dists, idxs); + } + + const int threads = 128; + const dim3 blocks(max_objects, batch_size); + size_t shared_size = threads * sizeof(size_t) + threads * sizeof(int64_t); + + DistanceForwardKernel<<>>( + objects.contiguous().data_ptr(), + objects_size, + objects_dim, + targets.contiguous().data_ptr(), + targets_size, + targets_dim, + objects_first_idx.contiguous().data_ptr(), + targets_first_idx.contiguous().data_ptr(), + batch_size, + dists.data_ptr(), + idxs.data_ptr(), + min_triangle_area); + + AT_CUDA_CHECK(cudaGetLastError()); + return std::make_tuple(dists, idxs); +} + +__global__ void DistanceBackwardKernel( + const float* __restrict__ objects, // (O * oD * 3) + const size_t objects_size, // O + const size_t objects_dim, // oD + const float* __restrict__ targets, // (T * tD * 3) + const size_t targets_dim, // tD + const int64_t* __restrict__ idx_objects, // (O,) + const float* __restrict__ grad_dists, // (O,) + float* __restrict__ grad_points, // ((O or T) * 3) + float* __restrict__ grad_face, // ((O or T) * max(oD, tD) * 3) + const double min_triangle_area) { + // This kernel is used interchangeably to compute bi-directional backward + // distances between points and triangles/lines. The direction of the distance + // computed, i.e. point to triangle/line or triangle/line to point, depends on + // the order of the input arguments and is inferred based on their shape. The + // shape is used to distinguish between triangles and lines. Note that + // grad_points will always be used for the point data and grad_face for the + // edge/triangle + + // Set references to points/face based on whether objects/targets are which + float3* points_f3 = objects_dim == 1 ? (float3*)objects : (float3*)targets; + float3* face_f3 = objects_dim == 1 ? (float3*)targets : (float3*)objects; + + const size_t tid = blockIdx.x * blockDim.x + threadIdx.x; + const size_t stride = gridDim.x * blockDim.x; + + for (size_t o = tid; o < objects_size; o += stride) { + const int64_t tidx = idx_objects[o]; + + size_t point_index = objects_dim == 1 ? o : tidx; + size_t face_index = objects_dim == 1 ? tidx * targets_dim : o * objects_dim; + bool isTriangle = objects_dim == 3 || targets_dim == 3; + + float3 grad_point, grad_v0, grad_v1, grad_v2; + if (isTriangle) { + const auto grads = PointTriangle3DistanceBackward( + points_f3[point_index], + face_f3[face_index], + face_f3[face_index + 1], + face_f3[face_index + 2], + grad_dists[o], + min_triangle_area); + grad_point = thrust::get<0>(grads); + grad_v0 = thrust::get<1>(grads); + grad_v1 = thrust::get<2>(grads); + grad_v2 = thrust::get<3>(grads); + } else { + const auto grads = PointLine3DistanceBackward( + points_f3[point_index], + face_f3[face_index], + face_f3[face_index + 1], + grad_dists[o]); + grad_point = thrust::get<0>(grads); + grad_v0 = thrust::get<1>(grads); + grad_v1 = thrust::get<2>(grads); + } + + atomicAdd(grad_points + point_index * 3 + 0, grad_point.x); + atomicAdd(grad_points + point_index * 3 + 1, grad_point.y); + atomicAdd(grad_points + point_index * 3 + 2, grad_point.z); + + atomicAdd(grad_face + face_index * 3 + 0 * 3 + 0, grad_v0.x); + atomicAdd(grad_face + face_index * 3 + 0 * 3 + 1, grad_v0.y); + atomicAdd(grad_face + face_index * 3 + 0 * 3 + 2, grad_v0.z); + + atomicAdd(grad_face + face_index * 3 + 1 * 3 + 0, grad_v1.x); + atomicAdd(grad_face + face_index * 3 + 1 * 3 + 1, grad_v1.y); + atomicAdd(grad_face + face_index * 3 + 1 * 3 + 2, grad_v1.z); + + if (isTriangle) { + atomicAdd(grad_face + face_index * 3 + 2 * 3 + 0, grad_v2.x); + atomicAdd(grad_face + face_index * 3 + 2 * 3 + 1, grad_v2.y); + atomicAdd(grad_face + face_index * 3 + 2 * 3 + 2, grad_v2.z); + } + } +} + +std::tuple DistanceBackwardCuda( + const at::Tensor& objects, + const size_t objects_dim, + const at::Tensor& targets, + const size_t targets_dim, + const at::Tensor& idx_objects, + const at::Tensor& grad_dists, + const double min_triangle_area) { + // Check inputs are on the same device + at::TensorArg objects_t{objects, "objects", 1}, + targets_t{targets, "targets", 2}, + idx_objects_t{idx_objects, "idx_objects", 3}, + grad_dists_t{grad_dists, "grad_dists", 4}; + at::CheckedFrom c = "DistanceBackwardCuda"; + at::checkAllSameGPU(c, {objects_t, targets_t, idx_objects_t, grad_dists_t}); + at::checkAllSameType(c, {objects_t, targets_t, grad_dists_t}); + // This is nondeterministic because atomicAdd + at::globalContext().alertNotDeterministic("DistanceBackwardCuda"); + + // Set the device for the kernel launch based on the device of the input + at::cuda::CUDAGuard device_guard(objects.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + const int64_t objects_size = objects.size(0); + const int64_t targets_size = targets.size(0); + + at::Tensor grad_points; + at::Tensor grad_tris; + + TORCH_CHECK(idx_objects.size(0) == objects_size); + TORCH_CHECK(grad_dists.size(0) == objects_size); + if (objects_dim == 1) { + TORCH_CHECK( + targets_dim >= 2 && targets_dim <= 3, + "either object or target must be edge or face"); + TORCH_CHECK(objects.size(1) == 3, "points must be of shape Px3"); + TORCH_CHECK( + targets.size(2) == 3, + "face must be of shape Tx3x3, lines must be of shape Tx2x3"); + // clang-format off + grad_points = at::zeros({objects_size, 3}, objects.options()); + grad_tris = at::zeros({targets_size, int64_t(targets_dim), 3}, targets.options()); + // clang-format on + } else { + TORCH_CHECK(targets_dim == 1, "either object or target must be point"); + TORCH_CHECK( + objects_dim >= 2 && objects_dim <= 3, + "either object or target must be edge or face"); + TORCH_CHECK(targets.size(1) == 3, "points must be of shape Px3"); + TORCH_CHECK( + objects.size(2) == 3, + "face must be of shape Tx3x3, lines must be of shape Tx2x3"); + // clang-format off + grad_points = at::zeros({targets_size, 3}, targets.options()); + grad_tris = at::zeros({objects_size, int64_t(objects_dim), 3}, objects.options()); + // clang-format on + } + + if (grad_points.numel() == 0 || grad_tris.numel() == 0) { + AT_CUDA_CHECK(cudaGetLastError()); + return std::make_tuple(grad_points, grad_tris); + } + + const int blocks = 64; + const int threads = 512; + + DistanceBackwardKernel<<>>( + objects.contiguous().data_ptr(), + objects_size, + objects_dim, + targets.contiguous().data_ptr(), + targets_dim, + idx_objects.contiguous().data_ptr(), + grad_dists.contiguous().data_ptr(), + grad_points.data_ptr(), + grad_tris.data_ptr(), + min_triangle_area); + + AT_CUDA_CHECK(cudaGetLastError()); + return std::make_tuple(grad_points, grad_tris); +} + +// **************************************************************************** +// * PointFaceDistance * +// **************************************************************************** + +std::tuple PointFaceDistanceForwardCuda( + const at::Tensor& points, + const at::Tensor& points_first_idx, + const at::Tensor& tris, + const at::Tensor& tris_first_idx, + const int64_t max_points, + const double min_triangle_area) { + return DistanceForwardCuda( + points, + 1, + points_first_idx, + tris, + 3, + tris_first_idx, + max_points, + min_triangle_area); +} + +std::tuple PointFaceDistanceBackwardCuda( + const at::Tensor& points, + const at::Tensor& tris, + const at::Tensor& idx_points, + const at::Tensor& grad_dists, + const double min_triangle_area) { + return DistanceBackwardCuda( + points, 1, tris, 3, idx_points, grad_dists, min_triangle_area); +} + +// **************************************************************************** +// * FacePointDistance * +// **************************************************************************** + +std::tuple FacePointDistanceForwardCuda( + const at::Tensor& points, + const at::Tensor& points_first_idx, + const at::Tensor& tris, + const at::Tensor& tris_first_idx, + const int64_t max_tris, + const double min_triangle_area) { + return DistanceForwardCuda( + tris, + 3, + tris_first_idx, + points, + 1, + points_first_idx, + max_tris, + min_triangle_area); +} + +std::tuple FacePointDistanceBackwardCuda( + const at::Tensor& points, + const at::Tensor& tris, + const at::Tensor& idx_tris, + const at::Tensor& grad_dists, + const double min_triangle_area) { + return DistanceBackwardCuda( + tris, 3, points, 1, idx_tris, grad_dists, min_triangle_area); +} + +// **************************************************************************** +// * PointEdgeDistance * +// **************************************************************************** + +std::tuple PointEdgeDistanceForwardCuda( + const at::Tensor& points, + const at::Tensor& points_first_idx, + const at::Tensor& segms, + const at::Tensor& segms_first_idx, + const int64_t max_points) { + return DistanceForwardCuda( + points, + 1, + points_first_idx, + segms, + 2, + segms_first_idx, + max_points, + 1); // todo: unused parameter handling for min_triangle_area +} + +std::tuple PointEdgeDistanceBackwardCuda( + const at::Tensor& points, + const at::Tensor& segms, + const at::Tensor& idx_points, + const at::Tensor& grad_dists) { + return DistanceBackwardCuda(points, 1, segms, 2, idx_points, grad_dists, 1); +} + +// **************************************************************************** +// * EdgePointDistance * +// **************************************************************************** + +std::tuple EdgePointDistanceForwardCuda( + const at::Tensor& points, + const at::Tensor& points_first_idx, + const at::Tensor& segms, + const at::Tensor& segms_first_idx, + const int64_t max_segms) { + return DistanceForwardCuda( + segms, 2, segms_first_idx, points, 1, points_first_idx, max_segms, 1); +} + +std::tuple EdgePointDistanceBackwardCuda( + const at::Tensor& points, + const at::Tensor& segms, + const at::Tensor& idx_segms, + const at::Tensor& grad_dists) { + return DistanceBackwardCuda(segms, 2, points, 1, idx_segms, grad_dists, 1); +} + +// **************************************************************************** +// * PointFaceArrayDistance * +// **************************************************************************** +// TODO: Create wrapper function and merge kernel with other array kernel + +__global__ void PointFaceArrayForwardKernel( + const float* __restrict__ points, // (P, 3) + const float* __restrict__ tris, // (T, 3, 3) + float* __restrict__ dists, // (P, T) + const size_t P, + const size_t T, + const double min_triangle_area) { + const float3* points_f3 = (float3*)points; + const float3* tris_f3 = (float3*)tris; + + // Parallelize over P * S computations + const int num_threads = gridDim.x * blockDim.x; + const int tid = blockIdx.x * blockDim.x + threadIdx.x; + + for (int t_i = tid; t_i < P * T; t_i += num_threads) { + const int t = t_i / P; // segment index. + const int p = t_i % P; // point index + const float3 v0 = tris_f3[t * 3 + 0]; + const float3 v1 = tris_f3[t * 3 + 1]; + const float3 v2 = tris_f3[t * 3 + 2]; + + const float3 point = points_f3[p]; + float dist = + PointTriangle3DistanceForward(point, v0, v1, v2, min_triangle_area); + dists[p * T + t] = dist; + } +} + +at::Tensor PointFaceArrayDistanceForwardCuda( + const at::Tensor& points, + const at::Tensor& tris, + const double min_triangle_area) { + // Check inputs are on the same device + at::TensorArg points_t{points, "points", 1}, tris_t{tris, "tris", 2}; + at::CheckedFrom c = "PointFaceArrayDistanceForwardCuda"; + at::checkAllSameGPU(c, {points_t, tris_t}); + at::checkAllSameType(c, {points_t, tris_t}); + + // Set the device for the kernel launch based on the device of the input + at::cuda::CUDAGuard device_guard(points.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + const int64_t P = points.size(0); + const int64_t T = tris.size(0); + + TORCH_CHECK(points.size(1) == 3, "points must be of shape Px3"); + TORCH_CHECK( + (tris.size(1) == 3) && (tris.size(2) == 3), + "tris must be of shape Tx3x3"); + + at::Tensor dists = at::zeros({P, T}, points.options()); + + if (dists.numel() == 0) { + AT_CUDA_CHECK(cudaGetLastError()); + return dists; + } + + const size_t blocks = 1024; + const size_t threads = 64; + + PointFaceArrayForwardKernel<<>>( + points.contiguous().data_ptr(), + tris.contiguous().data_ptr(), + dists.data_ptr(), + P, + T, + min_triangle_area); + + AT_CUDA_CHECK(cudaGetLastError()); + return dists; +} + +__global__ void PointFaceArrayBackwardKernel( + const float* __restrict__ points, // (P, 3) + const float* __restrict__ tris, // (T, 3, 3) + const float* __restrict__ grad_dists, // (P, T) + float* __restrict__ grad_points, // (P, 3) + float* __restrict__ grad_tris, // (T, 3, 3) + const size_t P, + const size_t T, + const double min_triangle_area) { + const float3* points_f3 = (float3*)points; + const float3* tris_f3 = (float3*)tris; + + // Parallelize over P * S computations + const int num_threads = gridDim.x * blockDim.x; + const int tid = blockIdx.x * blockDim.x + threadIdx.x; + + for (int t_i = tid; t_i < P * T; t_i += num_threads) { + const int t = t_i / P; // triangle index. + const int p = t_i % P; // point index + const float3 v0 = tris_f3[t * 3 + 0]; + const float3 v1 = tris_f3[t * 3 + 1]; + const float3 v2 = tris_f3[t * 3 + 2]; + + const float3 point = points_f3[p]; + + const float grad_dist = grad_dists[p * T + t]; + const auto grad = PointTriangle3DistanceBackward( + point, v0, v1, v2, grad_dist, min_triangle_area); + + const float3 grad_point = thrust::get<0>(grad); + const float3 grad_v0 = thrust::get<1>(grad); + const float3 grad_v1 = thrust::get<2>(grad); + const float3 grad_v2 = thrust::get<3>(grad); + + atomicAdd(grad_points + 3 * p + 0, grad_point.x); + atomicAdd(grad_points + 3 * p + 1, grad_point.y); + atomicAdd(grad_points + 3 * p + 2, grad_point.z); + + atomicAdd(grad_tris + t * 3 * 3 + 0 * 3 + 0, grad_v0.x); + atomicAdd(grad_tris + t * 3 * 3 + 0 * 3 + 1, grad_v0.y); + atomicAdd(grad_tris + t * 3 * 3 + 0 * 3 + 2, grad_v0.z); + + atomicAdd(grad_tris + t * 3 * 3 + 1 * 3 + 0, grad_v1.x); + atomicAdd(grad_tris + t * 3 * 3 + 1 * 3 + 1, grad_v1.y); + atomicAdd(grad_tris + t * 3 * 3 + 1 * 3 + 2, grad_v1.z); + + atomicAdd(grad_tris + t * 3 * 3 + 2 * 3 + 0, grad_v2.x); + atomicAdd(grad_tris + t * 3 * 3 + 2 * 3 + 1, grad_v2.y); + atomicAdd(grad_tris + t * 3 * 3 + 2 * 3 + 2, grad_v2.z); + } +} + +std::tuple PointFaceArrayDistanceBackwardCuda( + const at::Tensor& points, + const at::Tensor& tris, + const at::Tensor& grad_dists, + const double min_triangle_area) { + // Check inputs are on the same device + at::TensorArg points_t{points, "points", 1}, tris_t{tris, "tris", 2}, + grad_dists_t{grad_dists, "grad_dists", 3}; + at::CheckedFrom c = "PointFaceArrayDistanceBackwardCuda"; + at::checkAllSameGPU(c, {points_t, tris_t, grad_dists_t}); + at::checkAllSameType(c, {points_t, tris_t, grad_dists_t}); + // This is nondeterministic because atomicAdd + at::globalContext().alertNotDeterministic( + "PointFaceArrayDistanceBackwardCuda"); + + // Set the device for the kernel launch based on the device of the input + at::cuda::CUDAGuard device_guard(points.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + const int64_t P = points.size(0); + const int64_t T = tris.size(0); + + TORCH_CHECK(points.size(1) == 3, "points must be of shape Px3"); + TORCH_CHECK( + (tris.size(1) == 3) && (tris.size(2) == 3), + "tris must be of shape Tx3x3"); + TORCH_CHECK((grad_dists.size(0) == P) && (grad_dists.size(1) == T)); + + at::Tensor grad_points = at::zeros({P, 3}, points.options()); + at::Tensor grad_tris = at::zeros({T, 3, 3}, tris.options()); + + if (grad_points.numel() == 0 || grad_tris.numel() == 0) { + AT_CUDA_CHECK(cudaGetLastError()); + return std::make_tuple(grad_points, grad_tris); + } + + const size_t blocks = 1024; + const size_t threads = 64; + + PointFaceArrayBackwardKernel<<>>( + points.contiguous().data_ptr(), + tris.contiguous().data_ptr(), + grad_dists.contiguous().data_ptr(), + grad_points.data_ptr(), + grad_tris.data_ptr(), + P, + T, + min_triangle_area); + + AT_CUDA_CHECK(cudaGetLastError()); + return std::make_tuple(grad_points, grad_tris); +} + +// **************************************************************************** +// * PointEdgeArrayDistance * +// **************************************************************************** +// TODO: Create wrapper function and merge kernel with other array kernel + +__global__ void PointEdgeArrayForwardKernel( + const float* __restrict__ points, // (P, 3) + const float* __restrict__ segms, // (S, 2, 3) + float* __restrict__ dists, // (P, S) + const size_t P, + const size_t S) { + float3* points_f3 = (float3*)points; + float3* segms_f3 = (float3*)segms; + + // Parallelize over P * S computations + const int num_threads = gridDim.x * blockDim.x; + const int tid = blockIdx.x * blockDim.x + threadIdx.x; + + for (int t_i = tid; t_i < P * S; t_i += num_threads) { + const int s = t_i / P; // segment index. + const int p = t_i % P; // point index + float3 a = segms_f3[s * 2 + 0]; + float3 b = segms_f3[s * 2 + 1]; + + float3 point = points_f3[p]; + float dist = PointLine3DistanceForward(point, a, b); + dists[p * S + s] = dist; + } +} + +at::Tensor PointEdgeArrayDistanceForwardCuda( + const at::Tensor& points, + const at::Tensor& segms) { + // Check inputs are on the same device + at::TensorArg points_t{points, "points", 1}, segms_t{segms, "segms", 2}; + at::CheckedFrom c = "PointEdgeArrayDistanceForwardCuda"; + at::checkAllSameGPU(c, {points_t, segms_t}); + at::checkAllSameType(c, {points_t, segms_t}); + + // Set the device for the kernel launch based on the device of the input + at::cuda::CUDAGuard device_guard(points.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + const int64_t P = points.size(0); + const int64_t S = segms.size(0); + + TORCH_CHECK(points.size(1) == 3, "points must be of shape Px3"); + TORCH_CHECK( + (segms.size(1) == 2) && (segms.size(2) == 3), + "segms must be of shape Sx2x3"); + + at::Tensor dists = at::zeros({P, S}, points.options()); + + if (dists.numel() == 0) { + AT_CUDA_CHECK(cudaGetLastError()); + return dists; + } + + const size_t blocks = 1024; + const size_t threads = 64; + + PointEdgeArrayForwardKernel<<>>( + points.contiguous().data_ptr(), + segms.contiguous().data_ptr(), + dists.data_ptr(), + P, + S); + + AT_CUDA_CHECK(cudaGetLastError()); + return dists; +} + +__global__ void PointEdgeArrayBackwardKernel( + const float* __restrict__ points, // (P, 3) + const float* __restrict__ segms, // (S, 2, 3) + const float* __restrict__ grad_dists, // (P, S) + float* __restrict__ grad_points, // (P, 3) + float* __restrict__ grad_segms, // (S, 2, 3) + const size_t P, + const size_t S) { + float3* points_f3 = (float3*)points; + float3* segms_f3 = (float3*)segms; + + // Parallelize over P * S computations + const int num_threads = gridDim.x * blockDim.x; + const int tid = blockIdx.x * blockDim.x + threadIdx.x; + + for (int t_i = tid; t_i < P * S; t_i += num_threads) { + const int s = t_i / P; // segment index. + const int p = t_i % P; // point index + const float3 a = segms_f3[s * 2 + 0]; + const float3 b = segms_f3[s * 2 + 1]; + + const float3 point = points_f3[p]; + const float grad_dist = grad_dists[p * S + s]; + const auto grads = PointLine3DistanceBackward(point, a, b, grad_dist); + const float3 grad_point = thrust::get<0>(grads); + const float3 grad_a = thrust::get<1>(grads); + const float3 grad_b = thrust::get<2>(grads); + + atomicAdd(grad_points + p * 3 + 0, grad_point.x); + atomicAdd(grad_points + p * 3 + 1, grad_point.y); + atomicAdd(grad_points + p * 3 + 2, grad_point.z); + + atomicAdd(grad_segms + s * 2 * 3 + 0 * 3 + 0, grad_a.x); + atomicAdd(grad_segms + s * 2 * 3 + 0 * 3 + 1, grad_a.y); + atomicAdd(grad_segms + s * 2 * 3 + 0 * 3 + 2, grad_a.z); + + atomicAdd(grad_segms + s * 2 * 3 + 1 * 3 + 0, grad_b.x); + atomicAdd(grad_segms + s * 2 * 3 + 1 * 3 + 1, grad_b.y); + atomicAdd(grad_segms + s * 2 * 3 + 1 * 3 + 2, grad_b.z); + } +} + +std::tuple PointEdgeArrayDistanceBackwardCuda( + const at::Tensor& points, + const at::Tensor& segms, + const at::Tensor& grad_dists) { + // Check inputs are on the same device + at::TensorArg points_t{points, "points", 1}, segms_t{segms, "segms", 2}, + grad_dists_t{grad_dists, "grad_dists", 3}; + at::CheckedFrom c = "PointEdgeArrayDistanceBackwardCuda"; + at::checkAllSameGPU(c, {points_t, segms_t, grad_dists_t}); + at::checkAllSameType(c, {points_t, segms_t, grad_dists_t}); + // This is nondeterministic because atomicAdd + at::globalContext().alertNotDeterministic( + "PointEdgeArrayDistanceBackwardCuda"); + + // Set the device for the kernel launch based on the device of the input + at::cuda::CUDAGuard device_guard(points.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + const int64_t P = points.size(0); + const int64_t S = segms.size(0); + + TORCH_CHECK(points.size(1) == 3, "points must be of shape Px3"); + TORCH_CHECK( + (segms.size(1) == 2) && (segms.size(2) == 3), + "segms must be of shape Sx2x3"); + TORCH_CHECK((grad_dists.size(0) == P) && (grad_dists.size(1) == S)); + + at::Tensor grad_points = at::zeros({P, 3}, points.options()); + at::Tensor grad_segms = at::zeros({S, 2, 3}, segms.options()); + + if (grad_points.numel() == 0 || grad_segms.numel() == 0) { + AT_CUDA_CHECK(cudaGetLastError()); + return std::make_tuple(grad_points, grad_segms); + } + + const size_t blocks = 1024; + const size_t threads = 64; + + PointEdgeArrayBackwardKernel<<>>( + points.contiguous().data_ptr(), + segms.contiguous().data_ptr(), + grad_dists.contiguous().data_ptr(), + grad_points.data_ptr(), + grad_segms.data_ptr(), + P, + S); + AT_CUDA_CHECK(cudaGetLastError()); + return std::make_tuple(grad_points, grad_segms); +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/point_mesh/point_mesh_cuda.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/point_mesh/point_mesh_cuda.h new file mode 100644 index 0000000000000000000000000000000000000000..529dd5604c2ff4a4b84c590611adc4ef83edce4d --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/point_mesh/point_mesh_cuda.h @@ -0,0 +1,707 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once +#include +#include +#include +#include "utils/pytorch3d_cutils.h" + +// **************************************************************************** +// * PointFaceDistance * +// **************************************************************************** + +// Computes the squared euclidean distance of each p in points to it closest +// triangular face belonging to the corresponding mesh example in the batch of +// size N. +// +// Args: +// points: FloatTensor of shape (P, 3) +// points_first_idx: LongTensor of shape (N,) indicating the first point +// index for each example in the batch +// tris: FloatTensor of shape (T, 3, 3) of the triangular faces. The t-th +// triangular face is spanned by (tris[t, 0], tris[t, 1], tris[t, 2]) +// tris_first_idx: LongTensor of shape (N,) indicating the first face +// index for each example in the batch +// max_points: Scalar equal to max(P_i) for i in [0, N - 1] containing +// the maximum number of points in the batch and is used to set +// the block dimensions in the CUDA implementation. +// min_triangle_area: triangles less than this size are considered +// points/lines. +// +// Returns: +// dists: FloatTensor of shape (P,), where dists[p] is the minimum +// squared euclidean distance of points[p] to the faces in the same +// example in the batch. +// idxs: LongTensor of shape (P,), where idxs[p] is the index of the closest +// face in the batch. +// So, dists[p] = d(points[p], tris[idxs[p], 0], tris[idxs[p], 1], +// tris[idxs[p], 2]) where d(u, v0, v1, v2) is the distance of u from the +// face spanned by (v0, v1, v2) +// +// + +#ifdef WITH_CUDA + +std::tuple PointFaceDistanceForwardCuda( + const torch::Tensor& points, + const torch::Tensor& points_first_idx, + const torch::Tensor& tris, + const torch::Tensor& tris_first_idx, + const int64_t max_points, + const double min_triangle_area); +#endif + +std::tuple PointFaceDistanceForwardCpu( + const torch::Tensor& points, + const torch::Tensor& points_first_idx, + const torch::Tensor& tris, + const torch::Tensor& tris_first_idx, + const double min_triangle_area); + +std::tuple PointFaceDistanceForward( + const torch::Tensor& points, + const torch::Tensor& points_first_idx, + const torch::Tensor& tris, + const torch::Tensor& tris_first_idx, + const int64_t max_points, + const double min_triangle_area) { + if (points.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(points); + CHECK_CUDA(points_first_idx); + CHECK_CUDA(tris); + CHECK_CUDA(tris_first_idx); + return PointFaceDistanceForwardCuda( + points, + points_first_idx, + tris, + tris_first_idx, + max_points, + min_triangle_area); +#else + AT_ERROR("Not compiled with GPU support."); +#endif + } + return PointFaceDistanceForwardCpu( + points, points_first_idx, tris, tris_first_idx, min_triangle_area); +} + +// Backward pass for PointFaceDistance. +// +// Args: +// points: FloatTensor of shape (P, 3) +// tris: FloatTensor of shape (T, 3, 3) +// idx_points: LongTensor of shape (P,) containing the indices +// of the closest face in the example in the batch. +// This is computed by the forward pass +// grad_dists: FloatTensor of shape (P,) +// min_triangle_area: triangles less than this size are considered +// points/lines. +// +// Returns: +// grad_points: FloatTensor of shape (P, 3) +// grad_tris: FloatTensor of shape (T, 3, 3) +// + +#ifdef WITH_CUDA + +std::tuple PointFaceDistanceBackwardCuda( + const torch::Tensor& points, + const torch::Tensor& tris, + const torch::Tensor& idx_points, + const torch::Tensor& grad_dists, + const double min_triangle_area); +#endif +std::tuple PointFaceDistanceBackwardCpu( + const torch::Tensor& points, + const torch::Tensor& tris, + const torch::Tensor& idx_points, + const torch::Tensor& grad_dists, + const double min_triangle_area); + +std::tuple PointFaceDistanceBackward( + const torch::Tensor& points, + const torch::Tensor& tris, + const torch::Tensor& idx_points, + const torch::Tensor& grad_dists, + const double min_triangle_area) { + if (points.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(points); + CHECK_CUDA(tris); + CHECK_CUDA(idx_points); + CHECK_CUDA(grad_dists); + return PointFaceDistanceBackwardCuda( + points, tris, idx_points, grad_dists, min_triangle_area); +#else + AT_ERROR("Not compiled with GPU support."); +#endif + } + return PointFaceDistanceBackwardCpu( + points, tris, idx_points, grad_dists, min_triangle_area); +} + +// **************************************************************************** +// * FacePointDistance * +// **************************************************************************** + +// Computes the squared euclidean distance of each triangular face to its +// closest point belonging to the corresponding example in the batch of size N. +// +// Args: +// points: FloatTensor of shape (P, 3) +// points_first_idx: LongTensor of shape (N,) indicating the first point +// index for each example in the batch +// tris: FloatTensor of shape (T, 3, 3) of the triangular faces. The t-th +// triangular face is spanned by (tris[t, 0], tris[t, 1], tris[t, 2]) +// tris_first_idx: LongTensor of shape (N,) indicating the first face +// index for each example in the batch +// max_tris: Scalar equal to max(T_i) for i in [0, N - 1] containing +// the maximum number of faces in the batch and is used to set +// the block dimensions in the CUDA implementation. +// min_triangle_area: triangles less than this size are considered +// points/lines. +// +// Returns: +// dists: FloatTensor of shape (T,), where dists[t] is the minimum squared +// euclidean distance of t-th triangular face from the closest point in +// the batch. +// idxs: LongTensor of shape (T,), where idxs[t] is the index of the closest +// point in the batch. +// So, dists[t] = d(points[idxs[t]], tris[t, 0], tris[t, 1], tris[t, 2]) +// where d(u, v0, v1, v2) is the distance of u from the triangular face +// spanned by (v0, v1, v2) +// + +#ifdef WITH_CUDA + +std::tuple FacePointDistanceForwardCuda( + const torch::Tensor& points, + const torch::Tensor& points_first_idx, + const torch::Tensor& tris, + const torch::Tensor& tris_first_idx, + const int64_t max_tris, + const double min_triangle_area); +#endif + +std::tuple FacePointDistanceForwardCpu( + const torch::Tensor& points, + const torch::Tensor& points_first_idx, + const torch::Tensor& tris, + const torch::Tensor& tris_first_idx, + const double min_triangle_area); + +std::tuple FacePointDistanceForward( + const torch::Tensor& points, + const torch::Tensor& points_first_idx, + const torch::Tensor& tris, + const torch::Tensor& tris_first_idx, + const int64_t max_tris, + const double min_triangle_area) { + if (points.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(points); + CHECK_CUDA(points_first_idx); + CHECK_CUDA(tris); + CHECK_CUDA(tris_first_idx); + return FacePointDistanceForwardCuda( + points, + points_first_idx, + tris, + tris_first_idx, + max_tris, + min_triangle_area); +#else + AT_ERROR("Not compiled with GPU support."); +#endif + } + return FacePointDistanceForwardCpu( + points, points_first_idx, tris, tris_first_idx, min_triangle_area); +} + +// Backward pass for FacePointDistance. +// +// Args: +// points: FloatTensor of shape (P, 3) +// tris: FloatTensor of shape (T, 3, 3) +// idx_tris: LongTensor of shape (T,) containing the indices +// of the closest point in the example in the batch. +// This is computed by the forward pass +// grad_dists: FloatTensor of shape (T,) +// min_triangle_area: triangles less than this size are considered +// points/lines. +// +// Returns: +// grad_points: FloatTensor of shape (P, 3) +// grad_tris: FloatTensor of shape (T, 3, 3) +// + +#ifdef WITH_CUDA + +std::tuple FacePointDistanceBackwardCuda( + const torch::Tensor& points, + const torch::Tensor& tris, + const torch::Tensor& idx_tris, + const torch::Tensor& grad_dists, + const double min_triangle_area); +#endif + +std::tuple FacePointDistanceBackwardCpu( + const torch::Tensor& points, + const torch::Tensor& tris, + const torch::Tensor& idx_tris, + const torch::Tensor& grad_dists, + const double min_triangle_area); + +std::tuple FacePointDistanceBackward( + const torch::Tensor& points, + const torch::Tensor& tris, + const torch::Tensor& idx_tris, + const torch::Tensor& grad_dists, + const double min_triangle_area) { + if (points.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(points); + CHECK_CUDA(tris); + CHECK_CUDA(idx_tris); + CHECK_CUDA(grad_dists); + return FacePointDistanceBackwardCuda( + points, tris, idx_tris, grad_dists, min_triangle_area); +#else + AT_ERROR("Not compiled with GPU support."); +#endif + } + return FacePointDistanceBackwardCpu( + points, tris, idx_tris, grad_dists, min_triangle_area); +} + +// **************************************************************************** +// * PointEdgeDistance * +// **************************************************************************** + +// Computes the squared euclidean distance of each p in points to the closest +// mesh edge belonging to the corresponding example in the batch of size N. +// +// Args: +// points: FloatTensor of shape (P, 3) +// points_first_idx: LongTensor of shape (N,) indicating the first point +// index for each example in the batch +// segms: FloatTensor of shape (S, 2, 3) of edge segments. The s-th edge +// segment is spanned by (segms[s, 0], segms[s, 1]) +// segms_first_idx: LongTensor of shape (N,) indicating the first edge +// index for each example in the batch +// max_points: Scalar equal to max(P_i) for i in [0, N - 1] containing +// the maximum number of points in the batch and is used to set +// the grid dimensions in the CUDA implementation. +// +// Returns: +// dists: FloatTensor of shape (P,), where dists[p] is the squared euclidean +// distance of points[p] to the closest edge in the same example in the +// batch. +// idxs: LongTensor of shape (P,), where idxs[p] is the index of the closest +// edge in the batch. +// So, dists[p] = d(points[p], segms[idxs[p], 0], segms[idxs[p], 1]), +// where d(u, v0, v1) is the distance of u from the segment spanned by +// (v0, v1). +// + +#ifdef WITH_CUDA + +std::tuple PointEdgeDistanceForwardCuda( + const torch::Tensor& points, + const torch::Tensor& points_first_idx, + const torch::Tensor& segms, + const torch::Tensor& segms_first_idx, + const int64_t max_points); +#endif + +std::tuple PointEdgeDistanceForwardCpu( + const torch::Tensor& points, + const torch::Tensor& points_first_idx, + const torch::Tensor& segms, + const torch::Tensor& segms_first_idx, + const int64_t max_points); + +std::tuple PointEdgeDistanceForward( + const torch::Tensor& points, + const torch::Tensor& points_first_idx, + const torch::Tensor& segms, + const torch::Tensor& segms_first_idx, + const int64_t max_points) { + if (points.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(points); + CHECK_CUDA(points_first_idx); + CHECK_CUDA(segms); + CHECK_CUDA(segms_first_idx); + return PointEdgeDistanceForwardCuda( + points, points_first_idx, segms, segms_first_idx, max_points); +#else + AT_ERROR("Not compiled with GPU support."); +#endif + } + return PointEdgeDistanceForwardCpu( + points, points_first_idx, segms, segms_first_idx, max_points); +} + +// Backward pass for PointEdgeDistance. +// +// Args: +// points: FloatTensor of shape (P, 3) +// segms: FloatTensor of shape (S, 2, 3) +// idx_points: LongTensor of shape (P,) containing the indices +// of the closest edge in the example in the batch. +// This is computed by the forward pass. +// grad_dists: FloatTensor of shape (P,) +// +// Returns: +// grad_points: FloatTensor of shape (P, 3) +// grad_segms: FloatTensor of shape (S, 2, 3) +// + +#ifdef WITH_CUDA + +std::tuple PointEdgeDistanceBackwardCuda( + const torch::Tensor& points, + const torch::Tensor& segms, + const torch::Tensor& idx_points, + const torch::Tensor& grad_dists); +#endif + +std::tuple PointEdgeDistanceBackwardCpu( + const torch::Tensor& points, + const torch::Tensor& segms, + const torch::Tensor& idx_points, + const torch::Tensor& grad_dists); + +std::tuple PointEdgeDistanceBackward( + const torch::Tensor& points, + const torch::Tensor& segms, + const torch::Tensor& idx_points, + const torch::Tensor& grad_dists) { + if (points.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(points); + CHECK_CUDA(segms); + CHECK_CUDA(idx_points); + CHECK_CUDA(grad_dists); + return PointEdgeDistanceBackwardCuda(points, segms, idx_points, grad_dists); +#else + AT_ERROR("Not compiled with GPU support."); +#endif + } + return PointEdgeDistanceBackwardCpu(points, segms, idx_points, grad_dists); +} + +// **************************************************************************** +// * EdgePointDistance * +// **************************************************************************** + +// Computes the squared euclidean distance of each edge segment to the closest +// point belonging to the corresponding example in the batch of size N. +// +// Args: +// points: FloatTensor of shape (P, 3) +// points_first_idx: LongTensor of shape (N,) indicating the first point +// index for each example in the batch +// segms: FloatTensor of shape (S, 2, 3) of edge segments. The s-th edge +// segment is spanned by (segms[s, 0], segms[s, 1]) +// segms_first_idx: LongTensor of shape (N,) indicating the first edge +// index for each example in the batch +// max_segms: Scalar equal to max(S_i) for i in [0, N - 1] containing +// the maximum number of edges in the batch and is used to set +// the block dimensions in the CUDA implementation. +// +// Returns: +// dists: FloatTensor of shape (S,), where dists[s] is the squared +// euclidean distance of s-th edge to the closest point in the +// corresponding example in the batch. +// idxs: LongTensor of shape (S,), where idxs[s] is the index of the closest +// point in the example in the batch. +// So, dists[s] = d(points[idxs[s]], segms[s, 0], segms[s, 1]), where +// d(u, v0, v1) is the distance of u from the segment spanned by (v0, v1) +// +// + +#ifdef WITH_CUDA + +std::tuple EdgePointDistanceForwardCuda( + const torch::Tensor& points, + const torch::Tensor& points_first_idx, + const torch::Tensor& segms, + const torch::Tensor& segms_first_idx, + const int64_t max_segms); +#endif + +std::tuple EdgePointDistanceForwardCpu( + const torch::Tensor& points, + const torch::Tensor& points_first_idx, + const torch::Tensor& segms, + const torch::Tensor& segms_first_idx, + const int64_t max_segms); + +std::tuple EdgePointDistanceForward( + const torch::Tensor& points, + const torch::Tensor& points_first_idx, + const torch::Tensor& segms, + const torch::Tensor& segms_first_idx, + const int64_t max_segms) { + if (points.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(points); + CHECK_CUDA(points_first_idx); + CHECK_CUDA(segms); + CHECK_CUDA(segms_first_idx); + return EdgePointDistanceForwardCuda( + points, points_first_idx, segms, segms_first_idx, max_segms); +#else + AT_ERROR("Not compiled with GPU support."); +#endif + } + return EdgePointDistanceForwardCpu( + points, points_first_idx, segms, segms_first_idx, max_segms); +} + +// Backward pass for EdgePointDistance. +// +// Args: +// points: FloatTensor of shape (P, 3) +// segms: FloatTensor of shape (S, 2, 3) +// idx_segms: LongTensor of shape (S,) containing the indices +// of the closest point in the example in the batch. +// This is computed by the forward pass +// grad_dists: FloatTensor of shape (S,) +// +// Returns: +// grad_points: FloatTensor of shape (P, 3) +// grad_segms: FloatTensor of shape (S, 2, 3) +// + +#ifdef WITH_CUDA + +std::tuple EdgePointDistanceBackwardCuda( + const torch::Tensor& points, + const torch::Tensor& segms, + const torch::Tensor& idx_segms, + const torch::Tensor& grad_dists); +#endif + +std::tuple EdgePointDistanceBackwardCpu( + const torch::Tensor& points, + const torch::Tensor& segms, + const torch::Tensor& idx_segms, + const torch::Tensor& grad_dists); + +std::tuple EdgePointDistanceBackward( + const torch::Tensor& points, + const torch::Tensor& segms, + const torch::Tensor& idx_segms, + const torch::Tensor& grad_dists) { + if (points.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(points); + CHECK_CUDA(segms); + CHECK_CUDA(idx_segms); + CHECK_CUDA(grad_dists); + return EdgePointDistanceBackwardCuda(points, segms, idx_segms, grad_dists); +#else + AT_ERROR("Not compiled with GPU support."); +#endif + } + return EdgePointDistanceBackwardCpu(points, segms, idx_segms, grad_dists); +} + +// **************************************************************************** +// * PointFaceArrayDistance * +// **************************************************************************** + +// Computes the squared euclidean distance of each p in points to each +// triangular face spanned by (v0, v1, v2) in tris. +// +// Args: +// points: FloatTensor of shape (P, 3) +// tris: FloatTensor of shape (T, 3, 3) of the triangular faces. The t-th +// triangular face is spanned by (tris[t, 0], tris[t, 1], tris[t, 2]) +// min_triangle_area: triangles less than this size are considered +// points/lines. +// +// Returns: +// dists: FloatTensor of shape (P, T), where dists[p, t] is the squared +// euclidean distance of points[p] to the face spanned by (v0, v1, v2) +// where v0 = tris[t, 0], v1 = tris[t, 1] and v2 = tris[t, 2] +// +// For pointcloud and meshes of batch size N, this function requires N +// computations. The memory occupied is O(NPT) which can become quite large. +// For example, a medium sized batch with N = 32 with P = 10000 and T = 5000 +// will require for the forward pass 5.8G of memory to store dists. + +#ifdef WITH_CUDA + +torch::Tensor PointFaceArrayDistanceForwardCuda( + const torch::Tensor& points, + const torch::Tensor& tris, + const double min_triangle_area); +#endif + +torch::Tensor PointFaceArrayDistanceForwardCpu( + const torch::Tensor& points, + const torch::Tensor& tris, + const double min_triangle_area); + +torch::Tensor PointFaceArrayDistanceForward( + const torch::Tensor& points, + const torch::Tensor& tris, + const double min_triangle_area) { + if (points.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(points); + CHECK_CUDA(tris); + return PointFaceArrayDistanceForwardCuda(points, tris, min_triangle_area); +#else + AT_ERROR("Not compiled with GPU support."); +#endif + } + return PointFaceArrayDistanceForwardCpu(points, tris, min_triangle_area); +} + +// Backward pass for PointFaceArrayDistance. +// +// Args: +// points: FloatTensor of shape (P, 3) +// tris: FloatTensor of shape (T, 3, 3) +// grad_dists: FloatTensor of shape (P, T) +// min_triangle_area: triangles less than this size are considered +// points/lines. +// +// Returns: +// grad_points: FloatTensor of shape (P, 3) +// grad_tris: FloatTensor of shape (T, 3, 3) +// + +#ifdef WITH_CUDA +std::tuple PointFaceArrayDistanceBackwardCuda( + const torch::Tensor& points, + const torch::Tensor& tris, + const torch::Tensor& grad_dists, + const double min_triangle_area); +#endif +std::tuple PointFaceArrayDistanceBackwardCpu( + const torch::Tensor& points, + const torch::Tensor& tris, + const torch::Tensor& grad_dists, + const double min_triangle_area); + +std::tuple PointFaceArrayDistanceBackward( + const torch::Tensor& points, + const torch::Tensor& tris, + const torch::Tensor& grad_dists, + const double min_triangle_area) { + if (points.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(points); + CHECK_CUDA(tris); + CHECK_CUDA(grad_dists); + return PointFaceArrayDistanceBackwardCuda( + points, tris, grad_dists, min_triangle_area); +#else + AT_ERROR("Not compiled with GPU support."); +#endif + } + return PointFaceArrayDistanceBackwardCpu( + points, tris, grad_dists, min_triangle_area); +} + +// **************************************************************************** +// * PointEdgeArrayDistance * +// **************************************************************************** + +// Computes the squared euclidean distance of each p in points to each edge +// segment in segms. +// +// Args: +// points: FloatTensor of shape (P, 3) +// segms: FloatTensor of shape (S, 2, 3) of edge segments. The s-th +// edge segment is spanned by (segms[s, 0], segms[s, 1]) +// +// Returns: +// dists: FloatTensor of shape (P, S), where dists[p, s] is the squared +// euclidean distance of points[p] to the segment spanned by +// (segms[s, 0], segms[s, 1]) +// +// For pointcloud and meshes of batch size N, this function requires N +// computations. The memory occupied is O(NPS) which can become quite large. +// For example, a medium sized batch with N = 32 with P = 10000 and S = 5000 +// will require for the forward pass 5.8G of memory to store dists. + +#ifdef WITH_CUDA +torch::Tensor PointEdgeArrayDistanceForwardCuda( + const torch::Tensor& points, + const torch::Tensor& segms); +#endif + +torch::Tensor PointEdgeArrayDistanceForwardCpu( + const torch::Tensor& points, + const torch::Tensor& segms); + +torch::Tensor PointEdgeArrayDistanceForward( + const torch::Tensor& points, + const torch::Tensor& segms) { + if (points.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(points); + CHECK_CUDA(segms); + return PointEdgeArrayDistanceForwardCuda(points, segms); +#else + AT_ERROR("Not compiled with GPU support."); +#endif + } + return PointEdgeArrayDistanceForwardCpu(points, segms); +} + +// Backward pass for PointEdgeArrayDistance. +// +// Args: +// points: FloatTensor of shape (P, 3) +// segms: FloatTensor of shape (S, 2, 3) +// grad_dists: FloatTensor of shape (P, S) +// +// Returns: +// grad_points: FloatTensor of shape (P, 3) +// grad_segms: FloatTensor of shape (S, 2, 3) +// + +#ifdef WITH_CUDA + +std::tuple PointEdgeArrayDistanceBackwardCuda( + const torch::Tensor& points, + const torch::Tensor& segms, + const torch::Tensor& grad_dists); +#endif + +std::tuple PointEdgeArrayDistanceBackwardCpu( + const torch::Tensor& points, + const torch::Tensor& segms, + const torch::Tensor& grad_dists); + +std::tuple PointEdgeArrayDistanceBackward( + const torch::Tensor& points, + const torch::Tensor& segms, + const torch::Tensor& grad_dists) { + if (points.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(points); + CHECK_CUDA(segms); + CHECK_CUDA(grad_dists); + return PointEdgeArrayDistanceBackwardCuda(points, segms, grad_dists); +#else + AT_ERROR("Not compiled with GPU support."); +#endif + } + return PointEdgeArrayDistanceBackwardCpu(points, segms, grad_dists); +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/points_to_volumes/points_to_volumes.cu b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/points_to_volumes/points_to_volumes.cu new file mode 100644 index 0000000000000000000000000000000000000000..43d4ed55a3b0c0bb6e5b0256ef62084ccebfe660 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/points_to_volumes/points_to_volumes.cu @@ -0,0 +1,349 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include + +using at::PackedTensorAccessor64; +using at::RestrictPtrTraits; + +// A chunk of work is blocksize-many points. +// There are N clouds in the batch, and P points in each cloud. +// The number of potential chunks to do per cloud is (1+(P-1)/blocksize), +// which we call chunks_per_cloud. +// These (N*chunks_per_cloud) chunks are divided among the gridSize-many blocks. +// In block b, we work on chunks b, b+gridSize, b+2*gridSize etc . +// In chunk i, we work on cloud (i/chunks_per_cloud) on points starting from +// blocksize*(i%chunks_per_cloud). + +// Explanation of the calculation is in the cpp file. + +// EightDirections(t) runs t(a,b,c) for every combination of boolean a, b, c. +template +static __device__ void EightDirections(T&& t) { + t(false, false, false); + t(false, false, true); + t(false, true, false); + t(false, true, true); + t(true, false, false); + t(true, false, true); + t(true, true, false); + t(true, true, true); +} + +__global__ void PointsToVolumesForwardKernel( + const PackedTensorAccessor64 points_3d, + const PackedTensorAccessor64 points_features, + PackedTensorAccessor64 volume_densities, + PackedTensorAccessor64 volume_features, + PackedTensorAccessor64 grid_sizes, + PackedTensorAccessor64 mask, + const float point_weight, + const bool align_corners, + const bool splat, + const int64_t batch_size, + const int64_t P, + const int64_t n_features) { + const int64_t chunks_per_cloud = (1 + (P - 1) / blockDim.x); + const int64_t chunks_to_do = batch_size * chunks_per_cloud; + const int scale_offset = align_corners ? 1 : 0; + const float offset = align_corners ? 0 : 0.5; + for (int64_t chunk = blockIdx.x; chunk < chunks_to_do; chunk += gridDim.x) { + const int64_t batch_index = chunk / chunks_per_cloud; + const int64_t start_point = blockDim.x * (chunk % chunks_per_cloud); + int64_t point_idx = start_point + threadIdx.x; + if (point_idx >= P) { + continue; + } + if (mask[batch_index][point_idx] == 0) { + continue; + } + auto volume_densities_aa = volume_densities[batch_index][0]; + auto volume_features_aa = volume_features[batch_index]; + auto point = points_3d[batch_index][point_idx]; + auto point_features = points_features[batch_index][point_idx]; + const int64_t grid_size_x = grid_sizes[batch_index][2]; + const int64_t grid_size_y = grid_sizes[batch_index][1]; + const int64_t grid_size_z = grid_sizes[batch_index][0]; + auto increment_location = + [&](int64_t x, int64_t y, int64_t z, float weight) { + if (x >= grid_size_x || y >= grid_size_y || z >= grid_size_z) { + return; + } + if (x < 0 || y < 0 || z < 0) { + return; + } + + atomicAdd(&volume_densities_aa[z][y][x], weight * point_weight); + + for (int64_t feature_idx = 0; feature_idx < n_features; + ++feature_idx) { + atomicAdd( + &volume_features_aa[feature_idx][z][y][x], + point_features[feature_idx] * weight * point_weight); + } + }; + if (!splat) { + long x = std::lround( + (point[0] + 1) * 0.5 * (grid_size_x - scale_offset) - offset); + long y = std::lround( + (point[1] + 1) * 0.5 * (grid_size_y - scale_offset) - offset); + long z = std::lround( + (point[2] + 1) * 0.5 * (grid_size_z - scale_offset) - offset); + increment_location(x, y, z, 1); + } else { + float x = 0, y = 0, z = 0; + float rx = std::modf( + (point[0] + 1) * 0.5 * (grid_size_x - scale_offset) - offset, &x); + float ry = std::modf( + (point[1] + 1) * 0.5 * (grid_size_y - scale_offset) - offset, &y); + float rz = std::modf( + (point[2] + 1) * 0.5 * (grid_size_z - scale_offset) - offset, &z); + auto handle_point = [&](bool up_x, bool up_y, bool up_z) { + float weight = + (up_x ? rx : 1 - rx) * (up_y ? ry : 1 - ry) * (up_z ? rz : 1 - rz); + increment_location(x + up_x, y + up_y, z + up_z, weight); + }; + EightDirections(handle_point); + } + } +} + +void PointsToVolumesForwardCuda( + const at::Tensor& points_3d, + const at::Tensor& points_features, + const at::Tensor& volume_densities, + const at::Tensor& volume_features, + const at::Tensor& grid_sizes, + const at::Tensor& mask, + const float point_weight, + const bool align_corners, + const bool splat) { + // Check inputs are on the same device + at::TensorArg points_3d_t{points_3d, "points_3d", 1}, + points_features_t{points_features, "points_features", 2}, + volume_densities_t{volume_densities, "volume_densities", 3}, + volume_features_t{volume_features, "volume_features", 4}, + grid_sizes_t{grid_sizes, "grid_sizes", 5}, mask_t{mask, "mask", 6}; + at::CheckedFrom c = "PointsToVolumesForwardCuda"; + at::checkAllSameGPU( + c, + {points_3d_t, + points_features_t, + volume_densities_t, + volume_features_t, + grid_sizes_t, + mask_t}); + + // This is nondeterministic because atomicAdd + at::globalContext().alertNotDeterministic("PointsToVolumesForwardCuda"); + + // Set the device for the kernel launch based on the device of the input + at::cuda::CUDAGuard device_guard(points_3d.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + const int blocks = 1024; + const int threads = 32; + + const int64_t batch_size = points_3d.size(0); + const int64_t P = points_3d.size(1); + const int64_t n_features = points_features.size(2); + + PointsToVolumesForwardKernel<<>>( + points_3d.packed_accessor64(), + points_features.packed_accessor64(), + volume_densities.packed_accessor64(), + volume_features.packed_accessor64(), + grid_sizes.packed_accessor64(), + mask.packed_accessor64(), + point_weight, + align_corners, + splat, + batch_size, + P, + n_features); +} + +__global__ void PointsToVolumesBackwardKernel( + const PackedTensorAccessor64 points_3d, + const PackedTensorAccessor64 points_features, + const PackedTensorAccessor64 grid_sizes, + const PackedTensorAccessor64 mask, + PackedTensorAccessor64 grad_volume_densities, + PackedTensorAccessor64 grad_volume_features, + PackedTensorAccessor64 grad_points_3d, + PackedTensorAccessor64 grad_points_features, + const float point_weight, + const bool align_corners, + const bool splat, + const int64_t batch_size, + const int64_t P, + const int64_t n_features) { + const int64_t chunks_per_cloud = (1 + (P - 1) / blockDim.x); + const int64_t chunks_to_do = batch_size * chunks_per_cloud; + const int scale_offset = align_corners ? 1 : 0; + const float offset = align_corners ? 0 : 0.5; + // Note that the gradients belonging to each point are only touched by + // a single thread in one of our "chunks", which is in a single block. + // So unlike in the forward pass, there's no need for atomics here. + for (int64_t chunk = blockIdx.x; chunk < chunks_to_do; chunk += gridDim.x) { + const int64_t batch_index = chunk / chunks_per_cloud; + const int64_t start_point = blockDim.x * (chunk % chunks_per_cloud); + int64_t point_idx = start_point + threadIdx.x; + if (point_idx >= P) { + continue; + } + if (mask[batch_index][point_idx] == 0) { + continue; + } + auto point = points_3d[batch_index][point_idx]; + auto point_features = points_features[batch_index][point_idx]; + auto grad_point = grad_points_3d[batch_index][point_idx]; + auto grad_point_features = grad_points_features[batch_index][point_idx]; + auto grad_volume_densities_a = grad_volume_densities[batch_index][0]; + auto grad_volume_features_a = grad_volume_features[batch_index]; + const int64_t grid_size_x = grid_sizes[batch_index][2]; + const int64_t grid_size_y = grid_sizes[batch_index][1]; + const int64_t grid_size_z = grid_sizes[batch_index][0]; + + auto increment_location = + [&](int64_t x, int64_t y, int64_t z, float weight) { + if (x >= grid_size_x || y >= grid_size_y || z >= grid_size_z) { + return false; + } + if (x < 0 || y < 0 || z < 0) { + return false; + } + + // This is a forward line, for comparison + // volume_densities_aa[z][y][x] += weight * point_weight; + + for (int64_t feature_idx = 0; feature_idx < n_features; + ++feature_idx) { + // This is a forward line, for comparison + // volume_features_aa[feature_idx][z][y][x] += + // point_features[feature_idx] * weight * point_weight; + grad_point_features[feature_idx] += + grad_volume_features_a[feature_idx][z][y][x] * weight * + point_weight; + } + return true; + }; + + if (!splat) { + long x = std::lround( + (point[0] + 1) * 0.5 * (grid_size_x - scale_offset) - offset); + long y = std::lround( + (point[1] + 1) * 0.5 * (grid_size_y - scale_offset) - offset); + long z = std::lround( + (point[2] + 1) * 0.5 * (grid_size_z - scale_offset) - offset); + increment_location(x, y, z, 1); + } else { + float x = 0, y = 0, z = 0; + float rx = std::modf( + (point[0] + 1) * 0.5 * (grid_size_x - scale_offset) - offset, &x); + float ry = std::modf( + (point[1] + 1) * 0.5 * (grid_size_y - scale_offset) - offset, &y); + float rz = std::modf( + (point[2] + 1) * 0.5 * (grid_size_z - scale_offset) - offset, &z); + auto handle_point = [&](bool up_x, bool up_y, bool up_z) { + float weight_x = (up_x ? rx : 1 - rx); + float weight_y = (up_y ? ry : 1 - ry); + float weight_z = (up_z ? rz : 1 - rz); + float weight = weight_x * weight_y * weight_z; + if (increment_location(x + up_x, y + up_y, z + up_z, weight)) { + // weight * point_weight has been added to + // volume_densities_aa[z+up_z][y+up_y][x+up_x] + // Also for each feature_idx, + // point_features[feature_idx] * weight * point_weight + // has been added to + // volume_features_aa[feature_idx][z+up_z][y+up_y][x+up_x] + + double source_gradient = + grad_volume_densities_a[z + up_z][y + up_y][x + up_x]; + for (int64_t feature_idx = 0; feature_idx < n_features; + ++feature_idx) { + source_gradient += point_features[feature_idx] * + grad_volume_features_a[feature_idx][z + up_z][y + up_y] + [x + up_x]; + } + grad_point[0] += source_gradient * (up_x ? 1 : -1) * weight_y * + weight_z * 0.5 * (grid_size_x - scale_offset) * point_weight; + grad_point[1] += source_gradient * (up_y ? 1 : -1) * weight_x * + weight_z * 0.5 * (grid_size_y - scale_offset) * point_weight; + grad_point[2] += source_gradient * (up_z ? 1 : -1) * weight_x * + weight_y * 0.5 * (grid_size_z - scale_offset) * point_weight; + } + }; + EightDirections(handle_point); + } + } +} + +void PointsToVolumesBackwardCuda( + const at::Tensor& points_3d, + const at::Tensor& points_features, + const at::Tensor& grid_sizes, + const at::Tensor& mask, + const float point_weight, + const bool align_corners, + const bool splat, + const at::Tensor& grad_volume_densities, + const at::Tensor& grad_volume_features, + const at::Tensor& grad_points_3d, + const at::Tensor& grad_points_features) { + // Check inputs are on the same device + at::TensorArg points_3d_t{points_3d, "points_3d", 1}, + points_features_t{points_features, "points_features", 2}, + grid_sizes_t{grid_sizes, "grid_sizes", 3}, mask_t{mask, "mask", 4}, + grad_volume_densities_t{ + grad_volume_densities, "grad_volume_densities", 8}, + grad_volume_features_t{grad_volume_features, "grad_volume_features", 9}, + grad_points_3d_t{grad_points_3d, "grad_points_3d", 10}, + grad_points_features_t{grad_points_features, "grad_points_features", 11}; + + at::CheckedFrom c = "PointsToVolumesBackwardCuda"; + at::checkAllSameGPU( + c, + {points_3d_t, + points_features_t, + grid_sizes_t, + mask_t, + grad_volume_densities_t, + grad_volume_features_t, + grad_points_3d_t, + grad_points_features_t}); + + // Set the device for the kernel launch based on the device of the input + at::cuda::CUDAGuard device_guard(points_3d.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + const int blocks = 1024; + const int threads = 32; + + const int64_t batch_size = points_3d.size(0); + const int64_t P = points_3d.size(1); + const int64_t n_features = points_features.size(2); + + PointsToVolumesBackwardKernel<<>>( + points_3d.packed_accessor64(), + points_features.packed_accessor64(), + grid_sizes.packed_accessor64(), + mask.packed_accessor64(), + grad_volume_densities.packed_accessor64(), + grad_volume_features.packed_accessor64(), + grad_points_3d.packed_accessor64(), + grad_points_features.packed_accessor64(), + point_weight, + align_corners, + splat, + batch_size, + P, + n_features); +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/points_to_volumes/points_to_volumes.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/points_to_volumes/points_to_volumes.h new file mode 100644 index 0000000000000000000000000000000000000000..4c5eba3c9e55f7dc81f1df8a7e6698abdec33d70 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/points_to_volumes/points_to_volumes.h @@ -0,0 +1,198 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once +#include +#include +#include +#include +#include "utils/pytorch3d_cutils.h" + +/* + volume_features and volume_densities are modified in place. + + Args: + points_3d: Batch of 3D point cloud coordinates of shape + `(minibatch, N, 3)` where N is the number of points + in each point cloud. Coordinates have to be specified in the + local volume coordinates (ranging in [-1, 1]). + points_features: Features of shape `(minibatch, N, feature_dim)` + corresponding to the points of the input point cloud `points_3d`. + volume_features: Batch of input feature volumes + of shape `(minibatch, feature_dim, D, H, W)` + volume_densities: Batch of input feature volume densities + of shape `(minibatch, 1, D, H, W)`. Each voxel should + contain a non-negative number corresponding to its + opaqueness (the higher, the less transparent). + + grid_sizes: `LongTensor` of shape (minibatch, 3) representing the + spatial resolutions of each of the the non-flattened `volumes` + tensors. Note that the following has to hold: + `torch.prod(grid_sizes, dim=1)==N_voxels`. + + point_weight: A scalar controlling how much weight a single point has. + + mask: A binary mask of shape `(minibatch, N)` determining + which 3D points are going to be converted to the resulting + volume. Set to `None` if all points are valid. + + align_corners: as for grid_sample. + + splat: if true, trilinear interpolation. If false all the weight goes in + the nearest voxel. +*/ + +void PointsToVolumesForwardCpu( + const torch::Tensor& points_3d, + const torch::Tensor& points_features, + const torch::Tensor& volume_densities, + const torch::Tensor& volume_features, + const torch::Tensor& grid_sizes, + const torch::Tensor& mask, + float point_weight, + bool align_corners, + bool splat); + +void PointsToVolumesForwardCuda( + const torch::Tensor& points_3d, + const torch::Tensor& points_features, + const torch::Tensor& volume_densities, + const torch::Tensor& volume_features, + const torch::Tensor& grid_sizes, + const torch::Tensor& mask, + float point_weight, + bool align_corners, + bool splat); + +inline void PointsToVolumesForward( + const torch::Tensor& points_3d, + const torch::Tensor& points_features, + const torch::Tensor& volume_densities, + const torch::Tensor& volume_features, + const torch::Tensor& grid_sizes, + const torch::Tensor& mask, + float point_weight, + bool align_corners, + bool splat) { + if (points_3d.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(points_3d); + CHECK_CUDA(points_features); + CHECK_CUDA(volume_densities); + CHECK_CUDA(volume_features); + CHECK_CUDA(grid_sizes); + CHECK_CUDA(mask); + PointsToVolumesForwardCuda( + points_3d, + points_features, + volume_densities, + volume_features, + grid_sizes, + mask, + point_weight, + align_corners, + splat); + torch::autograd::increment_version(volume_features); + torch::autograd::increment_version(volume_densities); + return; +#else + AT_ERROR("Not compiled with GPU support."); +#endif + } + PointsToVolumesForwardCpu( + points_3d, + points_features, + volume_densities, + volume_features, + grid_sizes, + mask, + point_weight, + align_corners, + splat); +} + +// grad_points_3d and grad_points_features are modified in place. + +void PointsToVolumesBackwardCpu( + const torch::Tensor& points_3d, + const torch::Tensor& points_features, + const torch::Tensor& grid_sizes, + const torch::Tensor& mask, + float point_weight, + bool align_corners, + bool splat, + const torch::Tensor& grad_volume_densities, + const torch::Tensor& grad_volume_features, + const torch::Tensor& grad_points_3d, + const torch::Tensor& grad_points_features); + +void PointsToVolumesBackwardCuda( + const torch::Tensor& points_3d, + const torch::Tensor& points_features, + const torch::Tensor& grid_sizes, + const torch::Tensor& mask, + float point_weight, + bool align_corners, + bool splat, + const torch::Tensor& grad_volume_densities, + const torch::Tensor& grad_volume_features, + const torch::Tensor& grad_points_3d, + const torch::Tensor& grad_points_features); + +inline void PointsToVolumesBackward( + const torch::Tensor& points_3d, + const torch::Tensor& points_features, + const torch::Tensor& grid_sizes, + const torch::Tensor& mask, + float point_weight, + bool align_corners, + bool splat, + const torch::Tensor& grad_volume_densities, + const torch::Tensor& grad_volume_features, + const torch::Tensor& grad_points_3d, + const torch::Tensor& grad_points_features) { + if (points_3d.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(points_3d); + CHECK_CUDA(points_features); + CHECK_CUDA(grid_sizes); + CHECK_CUDA(mask); + CHECK_CUDA(grad_volume_densities); + CHECK_CUDA(grad_volume_features); + CHECK_CUDA(grad_points_3d); + CHECK_CUDA(grad_points_features); + PointsToVolumesBackwardCuda( + points_3d, + points_features, + grid_sizes, + mask, + point_weight, + align_corners, + splat, + grad_volume_densities, + grad_volume_features, + grad_points_3d, + grad_points_features); + return; +#else + AT_ERROR("Not compiled with GPU support."); +#endif + } + PointsToVolumesBackwardCpu( + points_3d, + points_features, + grid_sizes, + mask, + point_weight, + align_corners, + splat, + grad_volume_densities, + grad_volume_features, + grad_points_3d, + grad_points_features); +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/points_to_volumes/points_to_volumes_cpu.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/points_to_volumes/points_to_volumes_cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..900ea097a9a6e734c694305ca8f78b3e4d6eccbf --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/points_to_volumes/points_to_volumes_cpu.cpp @@ -0,0 +1,319 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include +#include +#include +#include + +// In the x direction, the location {0, ..., grid_size_x - 1} correspond to +// points px in [-1, 1]. There are two ways to do this. + +// If align_corners=True, px=-1 is the exact location 0 and px=1 is the exact +// location grid_size_x - 1. +// So the location of px is {(px + 1) * 0.5} * (grid_size_x - 1). +// Note that if you generate random points within the bounds you are less likely +// to hit the edge locations than other locations. +// This can be thought of as saying "location i" means a specific point. + +// If align_corners=False, px=-1 is half way between the exact location 0 and +// the non-existent location -1, i.e. location -0.5. +// Similarly px=1 is is half way between the exact location grid_size_x-1 and +// the non-existent location grid_size, i.e. the location grid_size_x - 0.5. +// So the location of px is ({(px + 1) * 0.5} * grid_size_x) - 0.5. +// Note that if you generate random points within the bounds you are equally +// likely to hit any location. +// This can be thought of as saying "location i" means the whole box from +// (i-0.5) to (i+0.5) + +// EightDirections(t) runs t(a,b,c) for every combination of boolean a, b, c. +template +static void EightDirections(T&& t) { + t(false, false, false); + t(false, false, true); + t(false, true, false); + t(false, true, true); + t(true, false, false); + t(true, false, true); + t(true, true, false); + t(true, true, true); +} + +void PointsToVolumesForwardCpu( + const torch::Tensor& points_3d, + const torch::Tensor& points_features, + const torch::Tensor& volume_densities, + const torch::Tensor& volume_features, + const torch::Tensor& grid_sizes, + const torch::Tensor& mask, + const float point_weight, + const bool align_corners, + const bool splat) { + const int64_t batch_size = points_3d.size(0); + const int64_t P = points_3d.size(1); + const int64_t n_features = points_features.size(2); + + // We unify the formula for the location of px in the comment above as + // ({(px + 1) * 0.5} * (grid_size_x-scale_offset)) - offset. + const int scale_offset = align_corners ? 1 : 0; + const float offset = align_corners ? 0 : 0.5; + + auto points_3d_a = points_3d.accessor(); + auto points_features_a = points_features.accessor(); + auto volume_densities_a = volume_densities.accessor(); + auto volume_features_a = volume_features.accessor(); + auto grid_sizes_a = grid_sizes.accessor(); + auto mask_a = mask.accessor(); + + // For each batch element + for (int64_t batch_idx = 0; batch_idx < batch_size; ++batch_idx) { + auto points_3d_aa = points_3d_a[batch_idx]; + auto points_features_aa = points_features_a[batch_idx]; + auto volume_densities_aa = volume_densities_a[batch_idx][0]; + auto volume_features_aa = volume_features_a[batch_idx]; + auto grid_sizes_aa = grid_sizes_a[batch_idx]; + auto mask_aa = mask_a[batch_idx]; + + const int64_t grid_size_x = grid_sizes_aa[2]; + const int64_t grid_size_y = grid_sizes_aa[1]; + const int64_t grid_size_z = grid_sizes_aa[0]; + + // For each point + for (int64_t point_idx = 0; point_idx < P; ++point_idx) { + // Ignore point if mask is 0 + if (mask_aa[point_idx] == 0) { + continue; + } + auto point = points_3d_aa[point_idx]; + auto point_features = points_features_aa[point_idx]; + + // Define how to increment a location in the volume by an amount. The need + // for this depends on the interpolation method: + // once per point for nearest, eight times for splat. + auto increment_location = + [&](int64_t x, int64_t y, int64_t z, float weight) { + if (x >= grid_size_x || y >= grid_size_y || z >= grid_size_z) { + return; + } + if (x < 0 || y < 0 || z < 0) { + return; + } + + volume_densities_aa[z][y][x] += weight * point_weight; + + for (int64_t feature_idx = 0; feature_idx < n_features; + ++feature_idx) { + volume_features_aa[feature_idx][z][y][x] += + point_features[feature_idx] * weight * point_weight; + } + }; + + if (!splat) { + // Increment the location nearest the point. + long x = std::lround( + (point[0] + 1) * 0.5 * (grid_size_x - scale_offset) - offset); + long y = std::lround( + (point[1] + 1) * 0.5 * (grid_size_y - scale_offset) - offset); + long z = std::lround( + (point[2] + 1) * 0.5 * (grid_size_z - scale_offset) - offset); + increment_location(x, y, z, 1); + } else { + // There are 8 locations around the point which we need to worry about. + // Their coordinates are (x or x+1, y or y+1, z or z+1). + // rx is a number between 0 and 1 for the proportion in the x direction: + // rx==0 means weight all on the lower bound, x, rx=1-eps means most + // weight on x+1. Ditto for ry and yz. + float x = 0, y = 0, z = 0; + float rx = std::modf( + (point[0] + 1) * 0.5 * (grid_size_x - scale_offset) - offset, &x); + float ry = std::modf( + (point[1] + 1) * 0.5 * (grid_size_y - scale_offset) - offset, &y); + float rz = std::modf( + (point[2] + 1) * 0.5 * (grid_size_z - scale_offset) - offset, &z); + // Define how to fractionally increment one of the 8 locations around + // the point. + auto handle_point = [&](bool up_x, bool up_y, bool up_z) { + float weight = (up_x ? rx : 1 - rx) * (up_y ? ry : 1 - ry) * + (up_z ? rz : 1 - rz); + increment_location(x + up_x, y + up_y, z + up_z, weight); + }; + // and do so. + EightDirections(handle_point); + } + } + } + torch::autograd::increment_version(volume_features); + torch::autograd::increment_version(volume_densities); +} + +// With nearest, the only smooth dependence is that volume features +// depend on points features. +// +// With splat, the dependencies are as follows, with gradients passing +// in the opposite direction. +// +// points_3d points_features +// │ │ │ +// │ │ │ +// │ └───────────┐ │ +// │ │ │ +// │ │ │ +// ▼ ▼ ▼ +// volume_densities volume_features + +// It is also the case that the input volume_densities and +// volume_features affect the corresponding outputs (they are +// modified in place). +// But the forward pass just increments these by a value which +// does not depend on them. So our autograd backwards pass needs +// to copy the gradient for each of those outputs to the +// corresponding input. We just do that in the Python layer. + +void PointsToVolumesBackwardCpu( + const torch::Tensor& points_3d, + const torch::Tensor& points_features, + const torch::Tensor& grid_sizes, + const torch::Tensor& mask, + const float point_weight, + const bool align_corners, + const bool splat, + const torch::Tensor& grad_volume_densities, + const torch::Tensor& grad_volume_features, + const torch::Tensor& grad_points_3d, + const torch::Tensor& grad_points_features) { + const int64_t batch_size = points_3d.size(0); + const int64_t P = points_3d.size(1); + const int64_t n_features = grad_points_features.size(2); + const int scale_offset = align_corners ? 1 : 0; + const float offset = align_corners ? 0 : 0.5; + + auto points_3d_a = points_3d.accessor(); + auto points_features_a = points_features.accessor(); + auto grid_sizes_a = grid_sizes.accessor(); + auto mask_a = mask.accessor(); + auto grad_volume_densities_a = grad_volume_densities.accessor(); + auto grad_volume_features_a = grad_volume_features.accessor(); + auto grad_points_3d_a = grad_points_3d.accessor(); + auto grad_points_features_a = grad_points_features.accessor(); + + // For each batch element + for (int64_t batch_idx = 0; batch_idx < batch_size; ++batch_idx) { + auto points_3d_aa = points_3d_a[batch_idx]; + auto points_features_aa = points_features_a[batch_idx]; + auto grid_sizes_aa = grid_sizes_a[batch_idx]; + auto mask_aa = mask_a[batch_idx]; + auto grad_volume_densities_aa = grad_volume_densities_a[batch_idx][0]; + auto grad_volume_features_aa = grad_volume_features_a[batch_idx]; + auto grad_points_3d_aa = grad_points_3d_a[batch_idx]; + auto grad_points_features_aa = grad_points_features_a[batch_idx]; + + const int64_t grid_size_x = grid_sizes_aa[2]; + const int64_t grid_size_y = grid_sizes_aa[1]; + const int64_t grid_size_z = grid_sizes_aa[0]; + + // For each point + for (int64_t point_idx = 0; point_idx < P; ++point_idx) { + if (mask_aa[point_idx] == 0) { + continue; + } + auto point = points_3d_aa[point_idx]; + auto point_features = points_features_aa[point_idx]; + auto grad_point_features = grad_points_features_aa[point_idx]; + auto grad_point = grad_points_3d_aa[point_idx]; + + // Define how to (backwards) increment a location in the point cloud, + // to take gradients to the features. + // We return false if the location does not really exist, so there was + // nothing to do. + // This happens once per point for nearest, eight times for splat. + auto increment_location = + [&](int64_t x, int64_t y, int64_t z, float weight) { + if (x >= grid_size_x || y >= grid_size_y || z >= grid_size_z) { + return false; + } + if (x < 0 || y < 0 || z < 0) { + return false; + } + + for (int64_t feature_idx = 0; feature_idx < n_features; + ++feature_idx) { + // This is a forward line, for comparison + // volume_features_aa[feature_idx][z][y][x] += + // point_features[feature_idx] * weight * point_weight; + grad_point_features[feature_idx] += + grad_volume_features_aa[feature_idx][z][y][x] * weight * + point_weight; + } + return true; + }; + + if (!splat) { + long x = std::lround( + (point[0] + 1) * 0.5 * (grid_size_x - scale_offset) - offset); + long y = std::lround( + (point[1] + 1) * 0.5 * (grid_size_y - scale_offset) - offset); + long z = std::lround( + (point[2] + 1) * 0.5 * (grid_size_z - scale_offset) - offset); + increment_location(x, y, z, 1); + } else { + float x = 0, y = 0, z = 0; + float rx = std::modf( + (point[0] + 1) * 0.5 * (grid_size_x - scale_offset) - offset, &x); + float ry = std::modf( + (point[1] + 1) * 0.5 * (grid_size_y - scale_offset) - offset, &y); + float rz = std::modf( + (point[2] + 1) * 0.5 * (grid_size_z - scale_offset) - offset, &z); + auto handle_point = [&](bool up_x, bool up_y, bool up_z) { + float weight_x = (up_x ? rx : 1 - rx); + float weight_y = (up_y ? ry : 1 - ry); + float weight_z = (up_z ? rz : 1 - rz); + float weight = weight_x * weight_y * weight_z; + // For each of the eight locations, we first increment the feature + // gradient. + if (increment_location(x + up_x, y + up_y, z + up_z, weight)) { + // If the location is a real location, we also (in this splat + // case) need to update the gradient w.r.t. the point position. + // - the amount in this location is controlled by the weight. + // There are two contributions: + // (1) The point position affects how much density we added + // to the location's density, so we have a contribution + // from grad_volume_density. Specifically, + // weight * point_weight has been added to + // volume_densities_aa[z+up_z][y+up_y][x+up_x] + // + // (2) The point position affects how much of each of the + // point's features were added to the corresponding feature + // of this location, so we have a contribution from + // grad_volume_features. Specifically, for each feature_idx, + // point_features[feature_idx] * weight * point_weight + // has been added to + // volume_features_aa[feature_idx][z+up_z][y+up_y][x+up_x] + + float source_gradient = + grad_volume_densities_aa[z + up_z][y + up_y][x + up_x]; + for (int64_t feature_idx = 0; feature_idx < n_features; + ++feature_idx) { + source_gradient += point_features[feature_idx] * + grad_volume_features_aa[feature_idx][z + up_z][y + up_y] + [x + up_x]; + } + grad_point[0] += source_gradient * (up_x ? 1 : -1) * weight_y * + weight_z * 0.5 * (grid_size_x - scale_offset) * point_weight; + grad_point[1] += source_gradient * (up_y ? 1 : -1) * weight_x * + weight_z * 0.5 * (grid_size_y - scale_offset) * point_weight; + grad_point[2] += source_gradient * (up_z ? 1 : -1) * weight_x * + weight_y * 0.5 * (grid_size_z - scale_offset) * point_weight; + } + }; + EightDirections(handle_point); + } + } + } +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/constants.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/constants.h new file mode 100644 index 0000000000000000000000000000000000000000..a2eee6217158d3a2e7a3e92a52e5afa4107494ab --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/constants.h @@ -0,0 +1,19 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_CONSTANTS_H_ +#define PULSAR_NATIVE_CONSTANTS_H_ + +#define EPS 1E-6 +#define FEPS 1E-6f +#define MAX_FLOAT 3.4E38f +#define MAX_INT 2147483647 +#define MAX_UINT 4294967295u +#define MAX_USHORT 65535u + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/README.md b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/README.md new file mode 100644 index 0000000000000000000000000000000000000000..60c5d07cba3b8d403693e9aa3db2a0b74f66c472 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/README.md @@ -0,0 +1,5 @@ +# CUDA device compilation units + +This folder contains `.cu` files to create compilation units +for device-specific functions. See `../include/README.md` for +more information. diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/commands.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/commands.h new file mode 100644 index 0000000000000000000000000000000000000000..00e6f37852169c6dd3ccaaf02d0381039fe2edbc --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/commands.h @@ -0,0 +1,505 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_CUDA_COMMANDS_H_ +#define PULSAR_NATIVE_CUDA_COMMANDS_H_ + +// Definitions for GPU commands. +#include +#include +namespace cg = cooperative_groups; + +#ifdef __DRIVER_TYPES_H__ +#ifndef DEVICE_RESET +#define DEVICE_RESET cudaDeviceReset(); +#endif +#else +#ifndef DEVICE_RESET +#define DEVICE_RESET +#endif +#endif + +#define HANDLECUDA(CMD) CMD +// handleCudaError((CMD), __FILE__, __LINE__) +inline void +handleCudaError(const cudaError_t err, const char* file, const int line) { + if (err != cudaSuccess) { +#ifndef __NVCC__ + fprintf( + stderr, + "%s(%i) : getLastCudaError() CUDA error :" + " (%d) %s.\n", + file, + line, + static_cast(err), + cudaGetErrorString(err)); + DEVICE_RESET + exit(1); +#endif + } +} +inline void +getLastCudaError(const char* errorMessage, const char* file, const int line) { + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err) { + fprintf(stderr, "Error: %s.", errorMessage); + handleCudaError(err, file, line); + } +} + +#define ALIGN(VAL) __align__(VAL) +#define SYNC() HANDLECUDE(cudaDeviceSynchronize()) +#define THREADFENCE_B() __threadfence_block() +#define SHFL_SYNC(a, b, c) __shfl_sync((a), (b), (c)) +#define SHARED __shared__ +#define ACTIVEMASK() __activemask() +#define BALLOT(mask, val) __ballot_sync((mask), val) +/** + * Find the cumulative sum within a warp up to the current + * thread lane, with each mask thread contributing base. + */ +template +DEVICE T +WARP_CUMSUM(const cg::coalesced_group& group, const uint& mask, const T& base) { + T ret = base; + T shfl_val; + shfl_val = __shfl_down_sync(mask, ret, 1u); // Deactivate the rightmost lane. + ret += (group.thread_rank() < 31) * shfl_val; + shfl_val = __shfl_down_sync(mask, ret, 2u); + ret += (group.thread_rank() < 30) * shfl_val; + shfl_val = __shfl_down_sync(mask, ret, 4u); // ...4 + ret += (group.thread_rank() < 28) * shfl_val; + shfl_val = __shfl_down_sync(mask, ret, 8u); // ...8 + ret += (group.thread_rank() < 24) * shfl_val; + shfl_val = __shfl_down_sync(mask, ret, 16u); // ...16 + ret += (group.thread_rank() < 16) * shfl_val; + return ret; +} + +template +DEVICE T +WARP_MAX(const cg::coalesced_group& group, const uint& mask, const T& base) { + T ret = base; + ret = max(ret, __shfl_down_sync(mask, ret, 16u)); + ret = max(ret, __shfl_down_sync(mask, ret, 8u)); + ret = max(ret, __shfl_down_sync(mask, ret, 4u)); + ret = max(ret, __shfl_down_sync(mask, ret, 2u)); + ret = max(ret, __shfl_down_sync(mask, ret, 1u)); + return ret; +} + +template +DEVICE T +WARP_SUM(const cg::coalesced_group& group, const uint& mask, const T& base) { + T ret = base; + ret = ret + __shfl_down_sync(mask, ret, 16u); + ret = ret + __shfl_down_sync(mask, ret, 8u); + ret = ret + __shfl_down_sync(mask, ret, 4u); + ret = ret + __shfl_down_sync(mask, ret, 2u); + ret = ret + __shfl_down_sync(mask, ret, 1u); + return ret; +} + +INLINE DEVICE float3 WARP_SUM_FLOAT3( + const cg::coalesced_group& group, + const uint& mask, + const float3& base) { + float3 ret = base; + ret.x = WARP_SUM(group, mask, base.x); + ret.y = WARP_SUM(group, mask, base.y); + ret.z = WARP_SUM(group, mask, base.z); + return ret; +} + +// Floating point. +// #define FMUL(a, b) __fmul_rn((a), (b)) +#define FMUL(a, b) ((a) * (b)) +#define FDIV(a, b) __fdiv_rn((a), (b)) +// #define FSUB(a, b) __fsub_rn((a), (b)) +#define FSUB(a, b) ((a) - (b)) +#define FADD(a, b) __fadd_rn((a), (b)) +#define FSQRT(a) __fsqrt_rn(a) +#define FEXP(a) fasterexp(a) +#define FLN(a) fasterlog(a) +#define FPOW(a, b) __powf((a), (b)) +#define FMAX(a, b) fmax((a), (b)) +#define FMIN(a, b) fmin((a), (b)) +#define FCEIL(a) ceilf(a) +#define FFLOOR(a) floorf(a) +#define FROUND(x) nearbyintf(x) +#define FSATURATE(x) __saturatef(x) +#define FABS(a) abs(a) +#define IASF(a, loc) (loc) = __int_as_float(a) +#define FASI(a, loc) (loc) = __float_as_int(a) +#define FABSLEQAS(a, b, c) \ + ((a) <= (b) ? FSUB((b), (a)) <= (c) : FSUB((a), (b)) < (c)) +/** Calculates x*y+z. */ +#define FMA(x, y, z) __fmaf_rn((x), (y), (z)) +#define I2F(a) __int2float_rn(a) +#define FRCP(x) __frcp_rn(x) +__device__ static float atomicMax(float* address, float val) { + int* address_as_i = (int*)address; + int old = *address_as_i, assumed; + do { + assumed = old; + old = ::atomicCAS( + address_as_i, + assumed, + __float_as_int(::fmaxf(val, __int_as_float(assumed)))); + } while (assumed != old); + return __int_as_float(old); +} +__device__ static float atomicMin(float* address, float val) { + int* address_as_i = (int*)address; + int old = *address_as_i, assumed; + do { + assumed = old; + old = ::atomicCAS( + address_as_i, + assumed, + __float_as_int(::fminf(val, __int_as_float(assumed)))); + } while (assumed != old); + return __int_as_float(old); +} +#define DMAX(a, b) FMAX(a, b) +#define DMIN(a, b) FMIN(a, b) +#define DSQRT(a) sqrt(a) +#define DSATURATE(a) DMIN(1., DMAX(0., (a))) +// half +#define HADD(a, b) __hadd((a), (b)) +#define HSUB2(a, b) __hsub2((a), (b)) +#define HMUL2(a, b) __hmul2((a), (b)) +#define HSQRT(a) hsqrt(a) + +// uint. +#define CLZ(VAL) __clz(VAL) +#define POPC(a) __popc(a) +// +// +// +// +// +// +// +// +// +#define ATOMICADD(PTR, VAL) atomicAdd((PTR), (VAL)) +#define ATOMICADD_F3(PTR, VAL) \ + ATOMICADD(&((PTR)->x), VAL.x); \ + ATOMICADD(&((PTR)->y), VAL.y); \ + ATOMICADD(&((PTR)->z), VAL.z); +#if (CUDART_VERSION >= 10000) && (__CUDA_ARCH__ >= 600) +#define ATOMICADD_B(PTR, VAL) atomicAdd_block((PTR), (VAL)) +#else +#define ATOMICADD_B(PTR, VAL) ATOMICADD(PTR, VAL) +#endif +// +// +// +// +// int. +#define IMIN(a, b) min((a), (b)) +#define IMAX(a, b) max((a), (b)) +#define IABS(a) abs(a) + +// Checks. +// like TORCH_CHECK_ARG in PyTorch > 1.10 +#define ARGCHECK(cond, argN, ...) \ + TORCH_CHECK(cond, "invalid argument ", argN, ": ", __VA_ARGS__) + +// Math. +#define NORM3DF(x, y, z) norm3df(x, y, z) +#define RNORM3DF(x, y, z) rnorm3df(x, y, z) + +// High level. +#define GET_SORT_WS_SIZE(RES_PTR, KEY_TYPE, VAL_TYPE, NUM_OBJECTS) \ + cub::DeviceRadixSort::SortPairsDescending( \ + (void*)NULL, \ + *(RES_PTR), \ + reinterpret_cast(NULL), \ + reinterpret_cast(NULL), \ + reinterpret_cast(NULL), \ + reinterpret_cast(NULL), \ + (NUM_OBJECTS)); +#define GET_REDUCE_WS_SIZE(RES_PTR, TYPE, REDUCE_OP, NUM_OBJECTS) \ + { \ + TYPE init = TYPE(); \ + cub::DeviceReduce::Reduce( \ + (void*)NULL, \ + *(RES_PTR), \ + (TYPE*)NULL, \ + (TYPE*)NULL, \ + (NUM_OBJECTS), \ + (REDUCE_OP), \ + init); \ + } +#define GET_SELECT_WS_SIZE( \ + RES_PTR, TYPE_SELECTOR, TYPE_SELECTION, NUM_OBJECTS) \ + { \ + cub::DeviceSelect::Flagged( \ + (void*)NULL, \ + *(RES_PTR), \ + (TYPE_SELECTION*)NULL, \ + (TYPE_SELECTOR*)NULL, \ + (TYPE_SELECTION*)NULL, \ + (int*)NULL, \ + (NUM_OBJECTS)); \ + } +#define GET_SUM_WS_SIZE(RES_PTR, TYPE_SUM, NUM_OBJECTS) \ + { \ + cub::DeviceReduce::Sum( \ + (void*)NULL, \ + *(RES_PTR), \ + (TYPE_SUM*)NULL, \ + (TYPE_SUM*)NULL, \ + NUM_OBJECTS); \ + } +#define GET_MM_WS_SIZE(RES_PTR, TYPE, NUM_OBJECTS) \ + { \ + TYPE init = TYPE(); \ + cub::DeviceReduce::Max( \ + (void*)NULL, *(RES_PTR), (TYPE*)NULL, (TYPE*)NULL, (NUM_OBJECTS)); \ + } +#define SORT_DESCENDING( \ + TMPN1, SORT_PTR, SORTED_PTR, VAL_PTR, VAL_SORTED_PTR, NUM_OBJECTS) \ + void* TMPN1 = NULL; \ + size_t TMPN1##_bytes = 0; \ + cub::DeviceRadixSort::SortPairsDescending( \ + TMPN1, \ + TMPN1##_bytes, \ + (SORT_PTR), \ + (SORTED_PTR), \ + (VAL_PTR), \ + (VAL_SORTED_PTR), \ + (NUM_OBJECTS)); \ + HANDLECUDA(cudaMalloc(&TMPN1, TMPN1##_bytes)); \ + cub::DeviceRadixSort::SortPairsDescending( \ + TMPN1, \ + TMPN1##_bytes, \ + (SORT_PTR), \ + (SORTED_PTR), \ + (VAL_PTR), \ + (VAL_SORTED_PTR), \ + (NUM_OBJECTS)); \ + HANDLECUDA(cudaFree(TMPN1)); +#define SORT_DESCENDING_WS( \ + TMPN1, \ + SORT_PTR, \ + SORTED_PTR, \ + VAL_PTR, \ + VAL_SORTED_PTR, \ + NUM_OBJECTS, \ + WORKSPACE_PTR, \ + WORKSPACE_BYTES) \ + cub::DeviceRadixSort::SortPairsDescending( \ + (WORKSPACE_PTR), \ + (WORKSPACE_BYTES), \ + (SORT_PTR), \ + (SORTED_PTR), \ + (VAL_PTR), \ + (VAL_SORTED_PTR), \ + (NUM_OBJECTS)); +#define SORT_ASCENDING_WS( \ + SORT_PTR, \ + SORTED_PTR, \ + VAL_PTR, \ + VAL_SORTED_PTR, \ + NUM_OBJECTS, \ + WORKSPACE_PTR, \ + WORKSPACE_BYTES, \ + STREAM) \ + cub::DeviceRadixSort::SortPairs( \ + (WORKSPACE_PTR), \ + (WORKSPACE_BYTES), \ + (SORT_PTR), \ + (SORTED_PTR), \ + (VAL_PTR), \ + (VAL_SORTED_PTR), \ + (NUM_OBJECTS), \ + 0, \ + sizeof(*(SORT_PTR)) * 8, \ + (STREAM)); +#define SUM_WS( \ + SUM_PTR, OUT_PTR, NUM_OBJECTS, WORKSPACE_PTR, WORKSPACE_BYTES, STREAM) \ + cub::DeviceReduce::Sum( \ + (WORKSPACE_PTR), \ + (WORKSPACE_BYTES), \ + (SUM_PTR), \ + (OUT_PTR), \ + (NUM_OBJECTS), \ + (STREAM)); +#define MIN_WS( \ + MIN_PTR, OUT_PTR, NUM_OBJECTS, WORKSPACE_PTR, WORKSPACE_BYTES, STREAM) \ + cub::DeviceReduce::Min( \ + (WORKSPACE_PTR), \ + (WORKSPACE_BYTES), \ + (MIN_PTR), \ + (OUT_PTR), \ + (NUM_OBJECTS), \ + (STREAM)); +#define MAX_WS( \ + MAX_PTR, OUT_PTR, NUM_OBJECTS, WORKSPACE_PTR, WORKSPACE_BYTES, STREAM) \ + cub::DeviceReduce::Min( \ + (WORKSPACE_PTR), \ + (WORKSPACE_BYTES), \ + (MAX_PTR), \ + (OUT_PTR), \ + (NUM_OBJECTS), \ + (STREAM)); +// +// +// +// TODO: rewrite using nested contexts instead of temporary names. +#define REDUCE(REDUCE_PTR, RESULT_PTR, NUM_ITEMS, REDUCE_OP, REDUCE_INIT) \ + cub::DeviceReduce::Reduce( \ + TMPN1, \ + TMPN1##_bytes, \ + (REDUCE_PTR), \ + (RESULT_PTR), \ + (NUM_ITEMS), \ + (REDUCE_OP), \ + (REDUCE_INIT)); \ + HANDLECUDA(cudaMalloc(&TMPN1, TMPN1##_bytes)); \ + cub::DeviceReduce::Reduce( \ + TMPN1, \ + TMPN1##_bytes, \ + (REDUCE_PTR), \ + (RESULT_PTR), \ + (NUM_ITEMS), \ + (REDUCE_OP), \ + (REDUCE_INIT)); \ + HANDLECUDA(cudaFree(TMPN1)); +#define REDUCE_WS( \ + REDUCE_PTR, \ + RESULT_PTR, \ + NUM_ITEMS, \ + REDUCE_OP, \ + REDUCE_INIT, \ + WORKSPACE_PTR, \ + WORSPACE_BYTES, \ + STREAM) \ + cub::DeviceReduce::Reduce( \ + (WORKSPACE_PTR), \ + (WORSPACE_BYTES), \ + (REDUCE_PTR), \ + (RESULT_PTR), \ + (NUM_ITEMS), \ + (REDUCE_OP), \ + (REDUCE_INIT), \ + (STREAM)); +#define SELECT_FLAGS_WS( \ + FLAGS_PTR, \ + ITEM_PTR, \ + OUT_PTR, \ + NUM_SELECTED_PTR, \ + NUM_ITEMS, \ + WORKSPACE_PTR, \ + WORSPACE_BYTES, \ + STREAM) \ + cub::DeviceSelect::Flagged( \ + (WORKSPACE_PTR), \ + (WORSPACE_BYTES), \ + (ITEM_PTR), \ + (FLAGS_PTR), \ + (OUT_PTR), \ + (NUM_SELECTED_PTR), \ + (NUM_ITEMS), \ + stream = (STREAM)); + +#define COPY_HOST_DEV(PTR_D, PTR_H, TYPE, SIZE) \ + HANDLECUDA(cudaMemcpy( \ + (PTR_D), (PTR_H), sizeof(TYPE) * (SIZE), cudaMemcpyHostToDevice)) +#define COPY_DEV_HOST(PTR_H, PTR_D, TYPE, SIZE) \ + HANDLECUDA(cudaMemcpy( \ + (PTR_H), (PTR_D), sizeof(TYPE) * (SIZE), cudaMemcpyDeviceToHost)) +#define COPY_DEV_DEV(PTR_T, PTR_S, TYPE, SIZE) \ + HANDLECUDA(cudaMemcpy( \ + (PTR_T), (PTR_S), sizeof(TYPE) * (SIZE), cudaMemcpyDeviceToDevice)) +// +// We *must* use cudaMallocManaged for pointers on device that should +// interact with pytorch. However, this comes at a significant speed penalty. +// We're using plain CUDA pointers for the rendering operations and +// explicitly copy results to managed pointers wrapped for pytorch (see +// pytorch/util.h). +#define MALLOC(VAR, TYPE, SIZE) cudaMalloc(&(VAR), sizeof(TYPE) * (SIZE)) +#define FREE(PTR) HANDLECUDA(cudaFree(PTR)) +#define MEMSET(VAR, VAL, TYPE, SIZE, STREAM) \ + HANDLECUDA(cudaMemsetAsync((VAR), (VAL), sizeof(TYPE) * (SIZE), (STREAM))) + +#define LAUNCH_MAX_PARALLEL_1D(FUNC, N, STREAM, ...) \ + { \ + int64_t max_threads = \ + at::cuda::getCurrentDeviceProperties()->maxThreadsPerBlock; \ + uint num_threads = min((N), max_threads); \ + uint num_blocks = iDivCeil((N), num_threads); \ + FUNC<<>>(__VA_ARGS__); \ + } +#define LAUNCH_PARALLEL_1D(FUNC, N, TN, STREAM, ...) \ + { \ + uint num_threads = min(static_cast(N), static_cast(TN)); \ + uint num_blocks = iDivCeil((N), num_threads); \ + FUNC<<>>(__VA_ARGS__); \ + } +#define LAUNCH_MAX_PARALLEL_2D(FUNC, NX, NY, STREAM, ...) \ + { \ + int64_t max_threads = \ + at::cuda::getCurrentDeviceProperties()->maxThreadsPerBlock; \ + int64_t max_threads_sqrt = static_cast(sqrt(max_threads)); \ + dim3 num_threads, num_blocks; \ + num_threads.x = min((NX), max_threads_sqrt); \ + num_blocks.x = iDivCeil((NX), num_threads.x); \ + num_threads.y = min((NY), max_threads_sqrt); \ + num_blocks.y = iDivCeil((NY), num_threads.y); \ + num_threads.z = 1; \ + num_blocks.z = 1; \ + FUNC<<>>(__VA_ARGS__); \ + } +#define LAUNCH_PARALLEL_2D(FUNC, NX, NY, TX, TY, STREAM, ...) \ + { \ + dim3 num_threads, num_blocks; \ + num_threads.x = min((NX), (TX)); \ + num_blocks.x = iDivCeil((NX), num_threads.x); \ + num_threads.y = min((NY), (TY)); \ + num_blocks.y = iDivCeil((NY), num_threads.y); \ + num_threads.z = 1; \ + num_blocks.z = 1; \ + FUNC<<>>(__VA_ARGS__); \ + } + +#define GET_PARALLEL_IDX_1D(VARNAME, N) \ + const uint VARNAME = __mul24(blockIdx.x, blockDim.x) + threadIdx.x; \ + if (VARNAME >= (N)) { \ + return; \ + } +#define GET_PARALLEL_IDS_2D(VAR_X, VAR_Y, WIDTH, HEIGHT) \ + const uint VAR_X = __mul24(blockIdx.x, blockDim.x) + threadIdx.x; \ + const uint VAR_Y = __mul24(blockIdx.y, blockDim.y) + threadIdx.y; \ + if (VAR_X >= (WIDTH) || VAR_Y >= (HEIGHT)) \ + return; +#define END_PARALLEL() +#define END_PARALLEL_NORET() +#define END_PARALLEL_2D_NORET() +#define END_PARALLEL_2D() +#define RETURN_PARALLEL() return +#define CHECKLAUNCH() C10_CUDA_CHECK(cudaGetLastError()); +#define ISONDEVICE true +#define SYNCDEVICE() HANDLECUDA(cudaDeviceSynchronize()) +#define START_TIME(TN) \ + cudaEvent_t __time_start_##TN, __time_stop_##TN; \ + cudaEventCreate(&__time_start_##TN); \ + cudaEventCreate(&__time_stop_##TN); \ + cudaEventRecord(__time_start_##TN); +#define STOP_TIME(TN) cudaEventRecord(__time_stop_##TN); +#define GET_TIME(TN, TOPTR) \ + cudaEventSynchronize(__time_stop_##TN); \ + cudaEventElapsedTime((TOPTR), __time_start_##TN, __time_stop_##TN); +#define START_TIME_CU(TN) START_TIME(CN) +#define STOP_TIME_CU(TN) STOP_TIME(TN) +#define GET_TIME_CU(TN, TOPTR) GET_TIME(TN, TOPTR) + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.backward.gpu.cu b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.backward.gpu.cu new file mode 100644 index 0000000000000000000000000000000000000000..e0da7b7020c0a3f5ae0647030282adf0e0103d39 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.backward.gpu.cu @@ -0,0 +1,9 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "../include/renderer.backward.instantiate.h" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.backward_dbg.gpu.cu b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.backward_dbg.gpu.cu new file mode 100644 index 0000000000000000000000000000000000000000..a95bb421d2d9b6bfec1a9286e035b042b0d9842c --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.backward_dbg.gpu.cu @@ -0,0 +1,9 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "../include/renderer.backward_dbg.instantiate.h" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.calc_gradients.gpu.cu b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.calc_gradients.gpu.cu new file mode 100644 index 0000000000000000000000000000000000000000..ff38b08e0dfe46e65a94039c8dec7da721d0421a --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.calc_gradients.gpu.cu @@ -0,0 +1,9 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "../include/renderer.calc_gradients.instantiate.h" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.calc_signature.gpu.cu b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.calc_signature.gpu.cu new file mode 100644 index 0000000000000000000000000000000000000000..81c72192eaa877038d9383cfdd0adf9a91e06f97 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.calc_signature.gpu.cu @@ -0,0 +1,9 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "../include/renderer.calc_signature.instantiate.h" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.construct.gpu.cu b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.construct.gpu.cu new file mode 100644 index 0000000000000000000000000000000000000000..67583511aec2a6bd4dd8670aeb809939a3d2e19c --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.construct.gpu.cu @@ -0,0 +1,9 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "../include/renderer.construct.instantiate.h" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.create_selector.gpu.cu b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.create_selector.gpu.cu new file mode 100644 index 0000000000000000000000000000000000000000..52e265bcb2ab8ca9e4d08d90d1dc4fef75294520 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.create_selector.gpu.cu @@ -0,0 +1,9 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "../include/renderer.create_selector.instantiate.h" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.destruct.gpu.cu b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.destruct.gpu.cu new file mode 100644 index 0000000000000000000000000000000000000000..e61be93fa4c4893e6c4800f71cf49ef81c717ff0 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.destruct.gpu.cu @@ -0,0 +1,9 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "../include/renderer.destruct.instantiate.h" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.fill_bg.gpu.cu b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.fill_bg.gpu.cu new file mode 100644 index 0000000000000000000000000000000000000000..6c7b1a48b675b1dbe69992c81a8cbb8c8861911e --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.fill_bg.gpu.cu @@ -0,0 +1,9 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "../include/renderer.fill_bg.instantiate.h" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.forward.gpu.cu b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.forward.gpu.cu new file mode 100644 index 0000000000000000000000000000000000000000..bfb42debeeaa7444daec94a88830c39825239170 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.forward.gpu.cu @@ -0,0 +1,9 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "../include/renderer.forward.instantiate.h" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.norm_cam_gradients.gpu.cu b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.norm_cam_gradients.gpu.cu new file mode 100644 index 0000000000000000000000000000000000000000..93d666324a4973f44ad4becbeecaf34e0c7b96e5 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.norm_cam_gradients.gpu.cu @@ -0,0 +1,9 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "../include/renderer.norm_cam_gradients.instantiate.h" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.norm_sphere_gradients.gpu.cu b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.norm_sphere_gradients.gpu.cu new file mode 100644 index 0000000000000000000000000000000000000000..65339caea11645e4b7ba99a0af77c21b4ae2f738 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.norm_sphere_gradients.gpu.cu @@ -0,0 +1,9 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "../include/renderer.norm_sphere_gradients.instantiate.h" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.render.gpu.cu b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.render.gpu.cu new file mode 100644 index 0000000000000000000000000000000000000000..eb46adbafbc1c2a60dfb21fa9ce222828e53e31b --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/cuda/renderer.render.gpu.cu @@ -0,0 +1,9 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "../include/renderer.render.instantiate.h" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/global.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/global.h new file mode 100644 index 0000000000000000000000000000000000000000..3cea957e1f09d32494bc6e644e8929a22534270d --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/global.h @@ -0,0 +1,118 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_GLOBAL_H +#define PULSAR_GLOBAL_H + +#include "./constants.h" +#ifndef WIN32 +#include +#endif + +#if defined(_WIN64) || defined(_WIN32) +#define uint unsigned int +#define ushort unsigned short +#endif + +#include "./logging.h" // <- include before torch/extension.h + +#define MAX_GRAD_SPHERES 128 + +#ifdef __CUDACC__ +#define INLINE __forceinline__ +#define HOST __host__ +#define DEVICE __device__ +#define GLOBAL __global__ +#define RESTRICT __restrict__ +#define DEBUGBREAK() +#ifdef __NVCC_DIAG_PRAGMA_SUPPORT__ +#pragma nv_diag_suppress 1866 +#pragma nv_diag_suppress 2941 +#pragma nv_diag_suppress 2951 +#pragma nv_diag_suppress 2967 +#else +#pragma diag_suppress = attribute_not_allowed +#pragma diag_suppress = 1866 +#pragma diag_suppress = 2941 +#pragma diag_suppress = 2951 +#pragma diag_suppress = 2967 +#endif +#else // __CUDACC__ +#define INLINE inline +#define HOST +#define DEVICE +#define GLOBAL +#define RESTRICT +#define DEBUGBREAK() std::raise(SIGINT) +// Don't care about pytorch warnings; they shouldn't clutter our warnings. +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Weverything" +#include +#pragma clang diagnostic pop +#ifdef WITH_CUDA +#include +#include +#else +#ifndef cudaStream_t +typedef void* cudaStream_t; +#endif +struct int2 { + int x, y; +}; +struct ushort2 { + unsigned short x, y; +}; +struct float2 { + float x, y; +}; +struct float3 { + float x, y, z; +}; +inline float3 make_float3(const float& x, const float& y, const float& z) { + float3 res; + res.x = x; + res.y = y; + res.z = z; + return res; +} +#endif +namespace py = pybind11; + +inline bool operator==(const float3& a, const float3& b) { + return a.x == b.x && a.y == b.y && a.z == b.z; +} +#endif // __CUDACC__ +#define IHD INLINE HOST DEVICE + +// An assertion command that can be used on host and device. +#ifdef PULSAR_ASSERTIONS +#ifdef __CUDACC__ +#define PASSERT(VAL) \ + if (!(VAL)) { \ + printf( \ + "Pulsar assertion failed in %s, line %d: %s.\n", \ + __FILE__, \ + __LINE__, \ + #VAL); \ + } +#else +#define PASSERT(VAL) \ + if (!(VAL)) { \ + printf( \ + "Pulsar assertion failed in %s, line %d: %s.\n", \ + __FILE__, \ + __LINE__, \ + #VAL); \ + std::raise(SIGINT); \ + } +#endif +#else +#define PASSERT(VAL) +#endif + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/README.md b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/README.md new file mode 100644 index 0000000000000000000000000000000000000000..34f1bade9134da24f4038425c4b50fe1fffc45dc --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/README.md @@ -0,0 +1,5 @@ +# Device-specific host compilation units + +This folder contains `.cpp` files to create compilation units +for device specific functions. See `../include/README.md` for +more information. diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/commands.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/commands.h new file mode 100644 index 0000000000000000000000000000000000000000..a48eaaa901d557874ed84e92751da76a5bcbf6c3 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/commands.h @@ -0,0 +1,391 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_COMMANDS_H_ +#define PULSAR_NATIVE_COMMANDS_H_ + +#ifdef _MSC_VER +#include +#define __builtin_popcount (int)__popcnt +#endif + +// Definitions for CPU commands. +// #include +// #include + +namespace cg { +struct coalesced_group { + INLINE uint thread_rank() const { + return 0u; + } + INLINE uint size() const { + return 1u; + } + INLINE uint ballot(uint val) const { + return static_cast(val > 0); + } +}; + +struct thread_block { + INLINE uint thread_rank() const { + return 0u; + } + INLINE uint size() const { + return 1u; + } + INLINE void sync() const {} +}; + +INLINE coalesced_group coalesced_threads() { + coalesced_group ret; + return ret; +} + +INLINE thread_block this_thread_block() { + thread_block ret; + return ret; +} +} // namespace cg +#define SHFL_SYNC(a, b, c) (b) +template +T WARP_CUMSUM( + const cg::coalesced_group& group, + const uint& mask, + const T& base) { + return base; +} + +template +DEVICE T +WARP_MAX(const cg::coalesced_group& group, const uint& mask, const T& base) { + return base; +} + +template +DEVICE T +WARP_SUM(const cg::coalesced_group& group, const uint& mask, const T& base) { + return base; +} + +INLINE DEVICE float3 WARP_SUM_FLOAT3( + const cg::coalesced_group& group, + const uint& mask, + const float3& base) { + return base; +} + +#define ACTIVEMASK() (1u << 31) +#define ALIGN(VAL) +#define SYNC() +#define THREADFENCE_B() +#define BALLOT(mask, val) (val != 0) +#define SHARED +// Floating point. +#define FMAX(a, b) std::fmax((a), (b)) +#define FMIN(a, b) std::fmin((a), (b)) +INLINE float atomicMax(float* address, float val) { + *address = std::max(*address, val); + return *address; +} +INLINE float atomicMin(float* address, float val) { + *address = std::min(*address, val); + return *address; +} +#define FMUL(a, b) ((a) * (b)) +#define FDIV(a, b) ((a) / (b)) +#define FSUB(a, b) ((a) - (b)) +#define FABSLEQAS(a, b, c) \ + ((a) <= (b) ? FSUB((b), (a)) <= (c) : FSUB((a), (b)) < (c)) +#define FADD(a, b) ((a) + (b)) +#define FSQRT(a) sqrtf(a) +#define FEXP(a) fasterexp(a) +#define FLN(a) fasterlog(a) +#define FPOW(a, b) powf((a), (b)) +#define FROUND(x) roundf(x) +#define FCEIL(a) ceilf(a) +#define FFLOOR(a) floorf(a) +#define FSATURATE(x) std::max(0.f, std::min(1.f, x)) +#define FABS(a) abs(a) +#define FMA(x, y, z) ((x) * (y) + (z)) +#define I2F(a) static_cast(a) +#define FRCP(x) (1.f / (x)) +#define IASF(x, loc) memcpy(&(loc), &(x), sizeof(x)) +#define FASI(x, loc) memcpy(&(loc), &(x), sizeof(x)) +#define DMAX(a, b) std::max((a), (b)) +#define DMIN(a, b) std::min((a), (b)) +#define DSATURATE(a) DMIN(1., DMAX(0., (a))) +#define DSQRT(a) sqrt(a) +// +// +// +// +// +// +// +// +// +// +// +// +// uint. +#define CLZ(VAL) _clz(VAL) +template +INLINE T ATOMICADD(T* address, T val) { + T old = *address; + *address += val; + return old; +} +template +INLINE void ATOMICADD_F3(T* address, T val) { + ATOMICADD(&(address->x), val.x); + ATOMICADD(&(address->y), val.y); + ATOMICADD(&(address->z), val.z); +} +#define ATOMICADD_B(a, b) ATOMICADD((a), (b)) +#define POPC(a) __builtin_popcount(a) + +// int. +#define IMIN(a, b) std::min((a), (b)) +#define IMAX(a, b) std::max((a), (b)) +#define IABS(a) abs(a) + +// Checks. +// like TORCH_CHECK_ARG in PyTorch > 1.10 +#define ARGCHECK(cond, argN, ...) \ + TORCH_CHECK(cond, "invalid argument ", argN, ": ", __VA_ARGS__) + +// Math. +#define NORM3DF(x, y, z) sqrtf(x* x + y * y + z * z) +#define RNORM3DF(x, y, z) (1.f / sqrtf(x * x + y * y + z * z)) + +// High level. +#define PREFETCH(PTR) +#define GET_SORT_WS_SIZE(RES_PTR, KEY_TYPE, VAL_TYPE, NUM_OBJECTS) \ + *(RES_PTR) = 0; +#define GET_REDUCE_WS_SIZE(RES_PTR, TYPE, REDUCE_OP, NUM_OBJECTS) \ + *(RES_PTR) = 0; +#define GET_SELECT_WS_SIZE( \ + RES_PTR, TYPE_SELECTOR, TYPE_SELECTION, NUM_OBJECTS) \ + *(RES_PTR) = 0; +#define GET_SUM_WS_SIZE(RES_PTR, TYPE_SUM, NUM_OBJECTS) *(RES_PTR) = 0; +#define GET_MM_WS_SIZE(RES_PTR, TYPE, NUM_OBJECTS) *(RES_PTR) = 0; + +#define SORT_DESCENDING( \ + TMPN1, SORT_PTR, SORTED_PTR, VAL_PTR, VAL_SORTED_PTR, NUM_OBJECTS) \ + std::vector TMPN1(NUM_OBJECTS); \ + std::iota(TMPN1.begin(), TMPN1.end(), 0); \ + const auto TMPN1##_val_ptr = (SORT_PTR); \ + std::sort( \ + TMPN1.begin(), TMPN1.end(), [&TMPN1##_val_ptr](size_t i1, size_t i2) { \ + return TMPN1##_val_ptr[i1] > TMPN1##_val_ptr[i2]; \ + }); \ + for (int i = 0; i < (NUM_OBJECTS); ++i) { \ + (SORTED_PTR)[i] = (SORT_PTR)[TMPN1[i]]; \ + } \ + for (int i = 0; i < (NUM_OBJECTS); ++i) { \ + (VAL_SORTED_PTR)[i] = (VAL_PTR)[TMPN1[i]]; \ + } + +#define SORT_ASCENDING( \ + SORT_PTR, SORTED_PTR, VAL_PTR, VAL_SORTED_PTR, NUM_OBJECTS, STREAM) \ + { \ + std::vector TMPN1(NUM_OBJECTS); \ + std::iota(TMPN1.begin(), TMPN1.end(), 0); \ + const auto TMPN1_val_ptr = (SORT_PTR); \ + std::sort( \ + TMPN1.begin(), \ + TMPN1.end(), \ + [&TMPN1_val_ptr](size_t i1, size_t i2) -> bool { \ + return TMPN1_val_ptr[i1] < TMPN1_val_ptr[i2]; \ + }); \ + for (int i = 0; i < (NUM_OBJECTS); ++i) { \ + (SORTED_PTR)[i] = (SORT_PTR)[TMPN1[i]]; \ + } \ + for (int i = 0; i < (NUM_OBJECTS); ++i) { \ + (VAL_SORTED_PTR)[i] = (VAL_PTR)[TMPN1[i]]; \ + } \ + } + +#define SORT_DESCENDING_WS( \ + TMPN1, \ + SORT_PTR, \ + SORTED_PTR, \ + VAL_PTR, \ + VAL_SORTED_PTR, \ + NUM_OBJECTS, \ + WORSPACE_PTR, \ + WORKSPACE_SIZE) \ + SORT_DESCENDING( \ + TMPN1, SORT_PTR, SORTED_PTR, VAL_PTR, VAL_SORTED_PTR, NUM_OBJECTS) + +#define SORT_ASCENDING_WS( \ + SORT_PTR, \ + SORTED_PTR, \ + VAL_PTR, \ + VAL_SORTED_PTR, \ + NUM_OBJECTS, \ + WORSPACE_PTR, \ + WORKSPACE_SIZE, \ + STREAM) \ + SORT_ASCENDING( \ + SORT_PTR, SORTED_PTR, VAL_PTR, VAL_SORTED_PTR, NUM_OBJECTS, STREAM) + +#define REDUCE(REDUCE_PTR, RESULT_PTR, NUM_ITEMS, REDUCE_OP, REDUCE_INIT) \ + { \ + *(RESULT_PTR) = (REDUCE_INIT); \ + for (int i = 0; i < (NUM_ITEMS); ++i) { \ + *(RESULT_PTR) = REDUCE_OP(*(RESULT_PTR), (REDUCE_PTR)[i]); \ + } \ + } +#define REDUCE_WS( \ + REDUCE_PTR, \ + RESULT_PTR, \ + NUM_ITEMS, \ + REDUCE_OP, \ + REDUCE_INIT, \ + WORKSPACE_PTR, \ + WORKSPACE_SIZE, \ + STREAM) \ + REDUCE(REDUCE_PTR, RESULT_PTR, NUM_ITEMS, REDUCE_OP, REDUCE_INIT) + +#define SELECT_FLAGS_WS( \ + FLAGS_PTR, \ + ITEM_PTR, \ + OUT_PTR, \ + NUM_SELECTED_PTR, \ + NUM_ITEMS, \ + WORKSPACE_PTR, \ + WORSPACE_BYTES, \ + STREAM) \ + { \ + *NUM_SELECTED_PTR = 0; \ + ptrdiff_t write_pos = 0; \ + for (int i = 0; i < NUM_ITEMS; ++i) { \ + if (FLAGS_PTR[i]) { \ + OUT_PTR[write_pos++] = ITEM_PTR[i]; \ + *NUM_SELECTED_PTR += 1; \ + } \ + } \ + } + +template +void SUM_WS( + T* SUM_PTR, + T* OUT_PTR, + size_t NUM_OBJECTS, + char* WORKSPACE_PTR, + size_t WORKSPACE_BYTES, + cudaStream_t STREAM) { + *(OUT_PTR) = T(); + for (int i = 0; i < (NUM_OBJECTS); ++i) { + *(OUT_PTR) = *(OUT_PTR) + (SUM_PTR)[i]; + } +} + +template +void MIN_WS( + T* MIN_PTR, + T* OUT_PTR, + size_t NUM_OBJECTS, + char* WORKSPACE_PTR, + size_t WORKSPACE_BYTES, + cudaStream_t STREAM) { + *(OUT_PTR) = T(); + for (int i = 0; i < (NUM_OBJECTS); ++i) { + *(OUT_PTR) = std::min(*(OUT_PTR), (MIN_PTR)[i]); + } +} + +template +void MAX_WS( + T* MAX_PTR, + T* OUT_PTR, + size_t NUM_OBJECTS, + char* WORKSPACE_PTR, + size_t WORKSPACE_BYTES, + cudaStream_t STREAM) { + *(OUT_PTR) = T(); + for (int i = 0; i < (NUM_OBJECTS); ++i) { + *(OUT_PTR) = std::max(*(OUT_PTR), (MAX_PTR)[i]); + } +} +// +// +// +// +#define COPY_HOST_DEV(PTR_D, PTR_H, TYPE, SIZE) \ + std::memcpy((PTR_D), (PTR_H), sizeof(TYPE) * (SIZE)) +// +#define COPY_DEV_HOST(PTR_H, PTR_D, TYPE, SIZE) \ + std::memcpy((PTR_H), (PTR_D), sizeof(TYPE) * (SIZE)) +// +#define COPY_DEV_DEV(PTR_T, PTR_S, TYPE, SIZE) \ + std::memcpy((PTR_T), (PTR_S), sizeof(TYPE) * SIZE) +// + +#define MALLOC(VAR, TYPE, SIZE) MALLOC_HOST(VAR, TYPE, SIZE) +#define FREE(PTR) FREE_HOST(PTR) +#define MEMSET(VAR, VAL, TYPE, SIZE, STREAM) \ + memset((VAR), (VAL), sizeof(TYPE) * (SIZE)) +// + +#define LAUNCH_MAX_PARALLEL_1D(FUNC, N, STREAM, ...) FUNC(__VA_ARGS__); +#define LAUNCH_PARALLEL_1D(FUNC, N, TN, STREAM, ...) FUNC(__VA_ARGS__); +#define LAUNCH_MAX_PARALLEL_2D(FUNC, NX, NY, STREAM, ...) FUNC(__VA_ARGS__); +#define LAUNCH_PARALLEL_2D(FUNC, NX, NY, TX, TY, STREAM, ...) FUNC(__VA_ARGS__); +// +// +// +// +// +#define GET_PARALLEL_IDX_1D(VARNAME, N) \ + for (uint VARNAME = 0; VARNAME < (N); ++VARNAME) { +#define GET_PARALLEL_IDS_2D(VAR_X, VAR_Y, WIDTH, HEIGHT) \ + int2 blockDim; \ + blockDim.x = 1; \ + blockDim.y = 1; \ + uint __parallel_2d_width = WIDTH; \ + uint __parallel_2d_height = HEIGHT; \ + for (uint VAR_Y = 0; VAR_Y < __parallel_2d_height; ++(VAR_Y)) { \ + for (uint VAR_X = 0; VAR_X < __parallel_2d_width; ++(VAR_X)) { +// +// +// +#define END_PARALLEL() \ + end_parallel :; \ + } +#define END_PARALLEL_NORET() } +#define END_PARALLEL_2D() \ + end_parallel :; \ + } \ + } +#define END_PARALLEL_2D_NORET() \ + } \ + } +#define RETURN_PARALLEL() goto end_parallel; +#define CHECKLAUNCH() +#define ISONDEVICE false +#define SYNCDEVICE() +#define START_TIME(TN) \ + auto __time_start_##TN = std::chrono::steady_clock::now(); +#define STOP_TIME(TN) auto __time_stop_##TN = std::chrono::steady_clock::now(); +#define GET_TIME(TN, TOPTR) \ + *TOPTR = std::chrono::duration_cast( \ + __time_stop_##TN - __time_start_##TN) \ + .count() +#define START_TIME_CU(TN) \ + cudaEvent_t __time_start_##TN, __time_stop_##TN; \ + cudaEventCreate(&__time_start_##TN); \ + cudaEventCreate(&__time_stop_##TN); \ + cudaEventRecord(__time_start_##TN); +#define STOP_TIME_CU(TN) cudaEventRecord(__time_stop_##TN); +#define GET_TIME_CU(TN, TOPTR) \ + cudaEventSynchronize(__time_stop_##TN); \ + cudaEventElapsedTime((TOPTR), __time_start_##TN, __time_stop_##TN); + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.backward.cpu.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.backward.cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e0da7b7020c0a3f5ae0647030282adf0e0103d39 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.backward.cpu.cpp @@ -0,0 +1,9 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "../include/renderer.backward.instantiate.h" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.backward_dbg.cpu.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.backward_dbg.cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a95bb421d2d9b6bfec1a9286e035b042b0d9842c --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.backward_dbg.cpu.cpp @@ -0,0 +1,9 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "../include/renderer.backward_dbg.instantiate.h" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.calc_gradients.cpu.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.calc_gradients.cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ff38b08e0dfe46e65a94039c8dec7da721d0421a --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.calc_gradients.cpu.cpp @@ -0,0 +1,9 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "../include/renderer.calc_gradients.instantiate.h" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.calc_signature.cpu.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.calc_signature.cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..81c72192eaa877038d9383cfdd0adf9a91e06f97 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.calc_signature.cpu.cpp @@ -0,0 +1,9 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "../include/renderer.calc_signature.instantiate.h" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.construct.cpu.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.construct.cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..67583511aec2a6bd4dd8670aeb809939a3d2e19c --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.construct.cpu.cpp @@ -0,0 +1,9 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "../include/renderer.construct.instantiate.h" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.create_selector.cpu.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.create_selector.cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..52e265bcb2ab8ca9e4d08d90d1dc4fef75294520 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.create_selector.cpu.cpp @@ -0,0 +1,9 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "../include/renderer.create_selector.instantiate.h" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.destruct.cpu.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.destruct.cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e61be93fa4c4893e6c4800f71cf49ef81c717ff0 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.destruct.cpu.cpp @@ -0,0 +1,9 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "../include/renderer.destruct.instantiate.h" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.fill_bg.cpu.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.fill_bg.cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6c7b1a48b675b1dbe69992c81a8cbb8c8861911e --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.fill_bg.cpu.cpp @@ -0,0 +1,9 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "../include/renderer.fill_bg.instantiate.h" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.forward.cpu.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.forward.cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bfb42debeeaa7444daec94a88830c39825239170 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.forward.cpu.cpp @@ -0,0 +1,9 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "../include/renderer.forward.instantiate.h" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.norm_cam_gradients.cpu.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.norm_cam_gradients.cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..93d666324a4973f44ad4becbeecaf34e0c7b96e5 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.norm_cam_gradients.cpu.cpp @@ -0,0 +1,9 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "../include/renderer.norm_cam_gradients.instantiate.h" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.norm_sphere_gradients.cpu.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.norm_sphere_gradients.cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..65339caea11645e4b7ba99a0af77c21b4ae2f738 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.norm_sphere_gradients.cpu.cpp @@ -0,0 +1,9 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "../include/renderer.norm_sphere_gradients.instantiate.h" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.render.cpu.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.render.cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..eb46adbafbc1c2a60dfb21fa9ce222828e53e31b --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/host/renderer.render.cpu.cpp @@ -0,0 +1,9 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "../include/renderer.render.instantiate.h" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/README.md b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e963ff043abdbbf88af350512f60fb70a02a4774 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/README.md @@ -0,0 +1,16 @@ +# The `include` folder + +This folder contains header files with implementations of several useful +algorithms. These implementations are usually done in files called `x.device.h` +and use macros that route every device specific command to the right +implementation (see `commands.h`). + +If you're using a device specific implementation, include `x.device.h`. +This gives you the high-speed, device specific implementation that lets +you work with all the details of the datastructure. All function calls are +inlined. If you need to work with the high-level interface and be able to +dynamically pick a device, only include `x.h`. The functions there are +templated with a boolean `DEV` flag and are instantiated in device specific +compilation units. You will not be able to use any other functions, but can +use `func(params)` to work on a CUDA device, or `func(params)` +to work on the host. diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/camera.device.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/camera.device.h new file mode 100644 index 0000000000000000000000000000000000000000..f003db31ba09e177f0119083fc00cb27fb019c0d --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/camera.device.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_INCLUDE_CAMERA_DEVICE_H_ +#define PULSAR_NATIVE_INCLUDE_CAMERA_DEVICE_H_ + +#include "../global.h" +#include "./camera.h" +#include "./commands.h" + +namespace pulsar { +IHD CamGradInfo::CamGradInfo() { + cam_pos = make_float3(0.f, 0.f, 0.f); + pixel_0_0_center = make_float3(0.f, 0.f, 0.f); + pixel_dir_x = make_float3(0.f, 0.f, 0.f); + pixel_dir_y = make_float3(0.f, 0.f, 0.f); +} +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/camera.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/camera.h new file mode 100644 index 0000000000000000000000000000000000000000..cbb583a14e7ee4349d11fea9faf8a6f09cb5e66c --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/camera.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_INCLUDE_CAMERA_H_ +#define PULSAR_NATIVE_INCLUDE_CAMERA_H_ + +#include +#include "../global.h" + +namespace pulsar { +/** + * Everything that's needed to raycast with our camera model. + */ +struct CamInfo { + float3 eye; /** Position in world coordinates. */ + float3 pixel_0_0_center; /** LUC center of pixel position in world + coordinates. */ + float3 pixel_dir_x; /** Direction for increasing x for one pixel to the next, + * in world coordinates. */ + float3 pixel_dir_y; /** Direction for increasing y for one pixel to the next, + * in world coordinates. */ + float3 sensor_dir_z; /** Normalized direction vector from eye through the + * sensor in z direction (optical axis). */ + float half_pixel_size; /** Half size of a pixel, in world coordinates. This + * must be consistent with pixel_dir_x and pixel_dir_y! + */ + float focal_length; /** The focal length, if applicable. */ + uint aperture_width; /** Full image width in px, possibly not fully used + * in case of a shifted principal point. */ + uint aperture_height; /** Full image height in px, possibly not fully used + * in case of a shifted principal point. */ + uint film_width; /** Resulting image width. */ + uint film_height; /** Resulting image height. */ + /** The top left coordinates (inclusive) of the film in the full aperture. */ + uint film_border_left, film_border_top; + int32_t principal_point_offset_x; /** Horizontal principal point offset. */ + int32_t principal_point_offset_y; /** Vertical principal point offset. */ + float min_dist; /** Minimum distance for a ball to be rendered. */ + float max_dist; /** Maximum distance for a ball to be rendered. */ + float norm_fac; /** 1 / (max_dist - min_dist), pre-computed. */ + /** The depth where to place the background, in normalized coordinates where + * 0. is the backmost depth and 1. the frontmost. */ + float background_normalization_depth; + /** The number of image content channels to use. Usually three. */ + uint n_channels; + /** Whether to use an orthogonal instead of a perspective projection. */ + bool orthogonal_projection; + /** Whether to use a right-handed system (inverts the z axis). */ + bool right_handed; +}; + +inline bool operator==(const CamInfo& a, const CamInfo& b) { + return a.film_width == b.film_width && a.film_height == b.film_height && + a.background_normalization_depth == b.background_normalization_depth && + a.n_channels == b.n_channels && + a.orthogonal_projection == b.orthogonal_projection && + a.right_handed == b.right_handed; +}; + +struct CamGradInfo { + HOST DEVICE CamGradInfo(); + float3 cam_pos; + float3 pixel_0_0_center; + float3 pixel_dir_x; + float3 pixel_dir_y; +}; + +// TODO: remove once https://github.com/NVlabs/cub/issues/172 is resolved. +struct IntWrapper { + int val; +}; + +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/closest_sphere_tracker.device.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/closest_sphere_tracker.device.h new file mode 100644 index 0000000000000000000000000000000000000000..a533dd0048e7f624af7c14a4017b19fde3accff5 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/closest_sphere_tracker.device.h @@ -0,0 +1,138 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_INCLUDE_CLOSEST_SPHERE_TRACKER_DEVICE_H_ +#define PULSAR_NATIVE_INCLUDE_CLOSEST_SPHERE_TRACKER_DEVICE_H_ + +#include "../global.h" + +namespace pulsar { +namespace Renderer { + +/** + * A facility to track the closest spheres to the camera. + * + * Their max number is defined by MAX_GRAD_SPHERES (this is defined in + * `pulsar/native/global.h`). This is done to keep the performance as high as + * possible because this struct needs to do updates continuously on the GPU. + */ +struct ClosestSphereTracker { + public: + IHD ClosestSphereTracker(const int& n_track) : n_hits(0), n_track(n_track) { + PASSERT(n_track < MAX_GRAD_SPHERES); + // Initialize the sphere IDs to -1 and the weights to 0. + for (int i = 0; i < n_track; ++i) { + this->most_important_sphere_ids[i] = -1; + this->closest_sphere_intersection_depths[i] = MAX_FLOAT; + } + }; + + IHD void track( + const uint& sphere_idx, + const float& intersection_depth, + const uint& coord_x, + const uint& coord_y) { + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_TRACKER_PIX, + "tracker|tracking sphere %u (depth: %f).\n", + sphere_idx, + intersection_depth); + for (int i = IMIN(this->n_hits, n_track) - 1; i >= -1; --i) { + if (i < 0 || + this->closest_sphere_intersection_depths[i] < intersection_depth) { + // Write position is i+1. + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_TRACKER_PIX, + "tracker|determined writing position: %d.\n", + i + 1); + if (i + 1 < n_track) { + // Shift every other sphere back. + for (int j = n_track - 1; j > i + 1; --j) { + this->closest_sphere_intersection_depths[j] = + this->closest_sphere_intersection_depths[j - 1]; + this->most_important_sphere_ids[j] = + this->most_important_sphere_ids[j - 1]; + } + this->closest_sphere_intersection_depths[i + 1] = intersection_depth; + this->most_important_sphere_ids[i + 1] = sphere_idx; + } + break; + } + } +#if PULSAR_LOG_TRACKER_PIX + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_TRACKER_PIX, + "tracker|sphere list after adding sphere %u:\n", + sphere_idx); + for (int i = 0; i < n_track; ++i) { + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_TRACKER_PIX, + "tracker|sphere %d: %d (depth: %f).\n", + i, + this->most_important_sphere_ids[i], + this->closest_sphere_intersection_depths[i]); + } +#endif // PULSAR_LOG_TRACKER_PIX + this->n_hits += 1; + } + + /** + * Get the number of hits registered. + */ + IHD int get_n_hits() const { + return this->n_hits; + } + + /** + * Get the idx closest sphere ID. + * + * For example, get_closest_sphere_id(0) gives the overall closest + * sphere id. + * + * This method is implemented for highly optimized scenarios and will *not* + * perform an index check at runtime if assertions are disabled. idx must be + * >=0 and < IMIN(n_hits, n_track) for a valid result, if it is >= + * n_hits it will return -1. + */ + IHD int get_closest_sphere_id(const int& idx) { + PASSERT(idx >= 0 && idx < n_track); + return this->most_important_sphere_ids[idx]; + } + + /** + * Get the idx closest sphere normalized_depth. + * + * For example, get_closest_sphere_depth(0) gives the overall closest + * sphere depth (normalized). + * + * This method is implemented for highly optimized scenarios and will *not* + * perform an index check at runtime if assertions are disabled. idx must be + * >=0 and < IMIN(n_hits, n_track) for a valid result, if it is >= + * n_hits it will return 1. + FEPS. + */ + IHD float get_closest_sphere_depth(const int& idx) { + PASSERT(idx >= 0 && idx < n_track); + return this->closest_sphere_intersection_depths[idx]; + } + + private: + /** The number of registered hits so far. */ + int n_hits; + /** The number of intersections to track. Must be (malloc(sizeof(TYPE) * (SIZE))) +#define FREE_HOST(PTR) free(PTR) + +/* Include command definitions depending on CPU or GPU use. */ + +#ifdef __CUDACC__ +// TODO: find out which compiler we're using here and use the suppression. +// #pragma push +// #pragma diag_suppress = 68 +#include +// #pragma pop +#include "../cuda/commands.h" +#else +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Weverything" +#pragma clang diagnostic pop +#include "../host/commands.h" +#endif + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/fastermath.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/fastermath.h new file mode 100644 index 0000000000000000000000000000000000000000..cae598f9c0a7f903b502702dcb62173c8841a3b8 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/fastermath.h @@ -0,0 +1,88 @@ +#ifndef PULSAR_NATIVE_INCLUDE_FASTERMATH_H_ +#define PULSAR_NATIVE_INCLUDE_FASTERMATH_H_ + +// @lint-ignore-every LICENSELINT +/*=====================================================================* + * Copyright (C) 2011 Paul Mineiro * + * All rights reserved. * + * * + * Redistribution and use in source and binary forms, with * + * or without modification, are permitted provided that the * + * following conditions are met: * + * * + * * Redistributions of source code must retain the * + * above copyright notice, this list of conditions and * + * the following disclaimer. * + * * + * * Redistributions in binary form must reproduce the * + * above copyright notice, this list of conditions and * + * the following disclaimer in the documentation and/or * + * other materials provided with the distribution. * + * * + * * Neither the name of Paul Mineiro nor the names * + * of other contributors may be used to endorse or promote * + * products derived from this software without specific * + * prior written permission. * + * * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND * + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, * + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER * + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, * + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE * + * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF * + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * + * POSSIBILITY OF SUCH DAMAGE. * + * * + * Contact: Paul Mineiro * + *=====================================================================*/ + +#include +#include "./commands.h" + +#ifdef __cplusplus +#define cast_uint32_t static_cast +#else +#define cast_uint32_t (uint32_t) +#endif + +IHD float fasterlog2(float x) { + union { + float f; + uint32_t i; + } vx = {x}; + float y = vx.i; + y *= 1.1920928955078125e-7f; + return y - 126.94269504f; +} + +IHD float fasterlog(float x) { + // return 0.69314718f * fasterlog2 (x); + union { + float f; + uint32_t i; + } vx = {x}; + float y = vx.i; + y *= 8.2629582881927490e-8f; + return y - 87.989971088f; +} + +IHD float fasterpow2(float p) { + float clipp = (p < -126) ? -126.0f : p; + union { + uint32_t i; + float f; + } v = {cast_uint32_t((1 << 23) * (clipp + 126.94269504f))}; + return v.f; +} + +IHD float fasterexp(float p) { + return fasterpow2(1.442695040f * p); +} + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/math.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/math.h new file mode 100644 index 0000000000000000000000000000000000000000..d77e2ee1aabb8607c706a7faaee052eb6531b557 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/math.h @@ -0,0 +1,157 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_IMPL_MATH_H_ +#define PULSAR_NATIVE_IMPL_MATH_H_ + +#include "./camera.h" +#include "./commands.h" +#include "./fastermath.h" + +/** + * Get the direction of val. + * + * Returns +1 if val is positive, -1 if val is zero or negative. + */ +IHD int sign_dir(const int& val) { + return -(static_cast((val <= 0)) << 1) + 1; +}; + +/** + * Get the direction of val. + * + * Returns +1 if val is positive, -1 if val is zero or negative. + */ +IHD float sign_dir(const float& val) { + return static_cast(1 - (static_cast((val <= 0)) << 1)); +}; + +/** + * Integer ceil division. + */ +IHD uint iDivCeil(uint a, uint b) { + return (a % b != 0) ? (a / b + 1) : (a / b); +} + +IHD float3 outer_product_sum(const float3& a) { + return make_float3( + a.x * a.x + a.x * a.y + a.x * a.z, + a.x * a.y + a.y * a.y + a.y * a.z, + a.x * a.z + a.y * a.z + a.z * a.z); +} + +// TODO: put intrinsics here. +IHD float3 operator+(const float3& a, const float3& b) { + return make_float3(a.x + b.x, a.y + b.y, a.z + b.z); +} + +IHD void operator+=(float3& a, const float3& b) { + a.x += b.x; + a.y += b.y; + a.z += b.z; +} + +IHD void operator-=(float3& a, const float3& b) { + a.x -= b.x; + a.y -= b.y; + a.z -= b.z; +} + +IHD void operator/=(float3& a, const float& b) { + a.x /= b; + a.y /= b; + a.z /= b; +} + +IHD void operator*=(float3& a, const float& b) { + a.x *= b; + a.y *= b; + a.z *= b; +} + +IHD float3 operator/(const float3& a, const float& b) { + return make_float3(a.x / b, a.y / b, a.z / b); +} + +IHD float3 operator-(const float3& a, const float3& b) { + return make_float3(a.x - b.x, a.y - b.y, a.z - b.z); +} + +IHD float3 operator*(const float3& a, const float& b) { + return make_float3(a.x * b, a.y * b, a.z * b); +} + +IHD float3 operator*(const float3& a, const float3& b) { + return make_float3(a.x * b.x, a.y * b.y, a.z * b.z); +} + +IHD float3 operator*(const float& a, const float3& b) { + return b * a; +} + +INLINE DEVICE float length(const float3& v) { + // TODO: benchmark what's faster. + return NORM3DF(v.x, v.y, v.z); + // return __fsqrt_rn(v.x * v.x + v.y * v.y + v.z * v.z); +} + +/** + * Left-hand multiplication of the constructed rotation matrix with the vector. + */ +IHD float3 rotate( + const float3& v, + const float3& dir_x, + const float3& dir_y, + const float3& dir_z) { + return make_float3( + dir_x.x * v.x + dir_x.y * v.y + dir_x.z * v.z, + dir_y.x * v.x + dir_y.y * v.y + dir_y.z * v.z, + dir_z.x * v.x + dir_z.y * v.y + dir_z.z * v.z); +} + +INLINE DEVICE float3 normalize(const float3& v) { + return v * RNORM3DF(v.x, v.y, v.z); +} + +INLINE DEVICE float dot(const float3& a, const float3& b) { + return FADD(FADD(FMUL(a.x, b.x), FMUL(a.y, b.y)), FMUL(a.z, b.z)); +} + +INLINE DEVICE float3 cross(const float3& a, const float3& b) { + // TODO: faster + return make_float3( + a.y * b.z - a.z * b.y, a.z * b.x - a.x * b.z, a.x * b.y - a.y * b.x); +} + +namespace pulsar { +IHD CamGradInfo operator+(const CamGradInfo& a, const CamGradInfo& b) { + CamGradInfo res; + res.cam_pos = a.cam_pos + b.cam_pos; + res.pixel_0_0_center = a.pixel_0_0_center + b.pixel_0_0_center; + res.pixel_dir_x = a.pixel_dir_x + b.pixel_dir_x; + res.pixel_dir_y = a.pixel_dir_y + b.pixel_dir_y; + return res; +} + +IHD CamGradInfo operator*(const CamGradInfo& a, const float& b) { + CamGradInfo res; + res.cam_pos = a.cam_pos * b; + res.pixel_0_0_center = a.pixel_0_0_center * b; + res.pixel_dir_x = a.pixel_dir_x * b; + res.pixel_dir_y = a.pixel_dir_y * b; + return res; +} + +IHD IntWrapper operator+(const IntWrapper& a, const IntWrapper& b) { + IntWrapper res; + res.val = a.val + b.val; + return res; +} +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.backward.device.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.backward.device.h new file mode 100644 index 0000000000000000000000000000000000000000..dcd9dd50fd1e83229073fc8e86b815d4da4c99f9 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.backward.device.h @@ -0,0 +1,189 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_RENDERER_BACKWARD_DEVICE_H_ +#define PULSAR_NATIVE_RENDERER_BACKWARD_DEVICE_H_ + +#include "./camera.device.h" +#include "./math.h" +#include "./renderer.h" + +namespace pulsar { +namespace Renderer { + +template +void backward( + Renderer* self, + const float* grad_im, + const float* image, + const float* forw_info, + const float* vert_pos, + const float* vert_col, + const float* vert_rad, + const CamInfo& cam, + const float& gamma, + float percent_allowed_difference, + const uint& max_n_hits, + const float* vert_opy_d, + const size_t& num_balls, + const uint& mode, + const bool& dif_pos, + const bool& dif_col, + const bool& dif_rad, + const bool& dif_cam, + const bool& dif_opy, + cudaStream_t stream) { + ARGCHECK(gamma > 0.f && gamma <= 1.f, 6, "gamma must be in [0., 1.]"); + ARGCHECK( + percent_allowed_difference >= 0.f && percent_allowed_difference <= 1.f, + 7, + "percent_allowed_difference must be in [0., 1.]"); + ARGCHECK(max_n_hits >= 1u, 8, "max_n_hits must be >= 1"); + ARGCHECK( + num_balls > 0 && num_balls <= self->max_num_balls, + 9, + "num_balls must be >0 and less than max num balls!"); + ARGCHECK( + cam.film_width == self->cam.film_width && + cam.film_height == self->cam.film_height, + 5, + "cam film size must agree"); + ARGCHECK(mode <= 1, 10, "mode must be <= 1!"); + if (percent_allowed_difference < EPS) { + LOG(WARNING) << "percent_allowed_difference < " << FEPS << "! Clamping to " + << FEPS << "."; + percent_allowed_difference = FEPS; + } + if (percent_allowed_difference > 1.f - FEPS) { + LOG(WARNING) << "percent_allowed_difference > " << (1.f - FEPS) + << "! Clamping to " << (1.f - FEPS) << "."; + percent_allowed_difference = 1.f - FEPS; + } + LOG_IF(INFO, PULSAR_LOG_RENDER) << "Rendering backward pass..."; + // Update camera. + self->cam.eye = cam.eye; + self->cam.pixel_0_0_center = cam.pixel_0_0_center - cam.eye; + self->cam.pixel_dir_x = cam.pixel_dir_x; + self->cam.pixel_dir_y = cam.pixel_dir_y; + self->cam.sensor_dir_z = cam.sensor_dir_z; + self->cam.half_pixel_size = cam.half_pixel_size; + self->cam.focal_length = cam.focal_length; + self->cam.aperture_width = cam.aperture_width; + self->cam.aperture_height = cam.aperture_height; + self->cam.min_dist = cam.min_dist; + self->cam.max_dist = cam.max_dist; + self->cam.norm_fac = cam.norm_fac; + self->cam.principal_point_offset_x = cam.principal_point_offset_x; + self->cam.principal_point_offset_y = cam.principal_point_offset_y; + self->cam.film_border_left = cam.film_border_left; + self->cam.film_border_top = cam.film_border_top; +#ifdef PULSAR_TIMINGS_ENABLED + START_TIME(calc_signature); +#endif + LAUNCH_MAX_PARALLEL_1D( + calc_signature, + num_balls, + stream, + *self, + reinterpret_cast(vert_pos), + vert_col, + vert_rad, + num_balls); + CHECKLAUNCH(); +#ifdef PULSAR_TIMINGS_ENABLED + STOP_TIME(calc_signature); + START_TIME(calc_gradients); +#endif + MEMSET(self->grad_pos_d, 0, float3, num_balls, stream); + MEMSET(self->grad_col_d, 0, float, num_balls * self->cam.n_channels, stream); + MEMSET(self->grad_rad_d, 0, float, num_balls, stream); + MEMSET(self->grad_cam_d, 0, float, 12, stream); + MEMSET(self->grad_cam_buf_d, 0, CamGradInfo, num_balls, stream); + MEMSET(self->grad_opy_d, 0, float, num_balls, stream); + MEMSET(self->ids_sorted_d, 0, int, num_balls, stream); + LAUNCH_PARALLEL_2D( + calc_gradients, + self->cam.film_width, + self->cam.film_height, + GRAD_BLOCK_SIZE, + GRAD_BLOCK_SIZE, + stream, + self->cam, + grad_im, + gamma, + reinterpret_cast(vert_pos), + vert_col, + vert_rad, + vert_opy_d, + num_balls, + image, + forw_info, + self->di_d, + self->ii_d, + dif_pos, + dif_col, + dif_rad, + dif_cam, + dif_opy, + self->grad_rad_d, + self->grad_col_d, + self->grad_pos_d, + self->grad_cam_buf_d, + self->grad_opy_d, + self->ids_sorted_d, + self->n_track); + CHECKLAUNCH(); +#ifdef PULSAR_TIMINGS_ENABLED + STOP_TIME(calc_gradients); + START_TIME(normalize); +#endif + LAUNCH_MAX_PARALLEL_1D( + norm_sphere_gradients, num_balls, stream, *self, num_balls); + CHECKLAUNCH(); + if (dif_cam) { + SUM_WS( + self->grad_cam_buf_d, + reinterpret_cast(self->grad_cam_d), + static_cast(num_balls), + self->workspace_d, + self->workspace_size, + stream); + CHECKLAUNCH(); + SUM_WS( + (IntWrapper*)(self->ids_sorted_d), + (IntWrapper*)(self->n_grad_contributions_d), + static_cast(num_balls), + self->workspace_d, + self->workspace_size, + stream); + CHECKLAUNCH(); + LAUNCH_MAX_PARALLEL_1D( + norm_cam_gradients, static_cast(1), stream, *self); + CHECKLAUNCH(); + } +#ifdef PULSAR_TIMINGS_ENABLED + STOP_TIME(normalize); + float time_ms; + // This blocks the result and prevents batch-processing from parallelizing. + GET_TIME(calc_signature, &time_ms); + std::cout << "Time for signature calculation: " << time_ms << " ms" + << std::endl; + GET_TIME(calc_gradients, &time_ms); + std::cout << "Time for gradient calculation: " << time_ms << " ms" + << std::endl; + GET_TIME(normalize, &time_ms); + std::cout << "Time for aggregation and normalization: " << time_ms << " ms" + << std::endl; +#endif + LOG_IF(INFO, PULSAR_LOG_RENDER) << "Backward pass complete."; +} + +} // namespace Renderer +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.backward.instantiate.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.backward.instantiate.h new file mode 100644 index 0000000000000000000000000000000000000000..75e85fd4dc88befc63e84ed4891f8ecb4b659bc4 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.backward.instantiate.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "./renderer.backward.device.h" + +namespace pulsar { +namespace Renderer { + +template void backward( + Renderer* self, + const float* grad_im, + const float* image, + const float* forw_info, + const float* vert_pos, + const float* vert_col, + const float* vert_rad, + const CamInfo& cam, + const float& gamma, + float percent_allowed_difference, + const uint& max_n_hits, + const float* vert_opy, + const size_t& num_balls, + const uint& mode, + const bool& dif_pos, + const bool& dif_col, + const bool& dif_rad, + const bool& dif_cam, + const bool& dif_opy, + cudaStream_t stream); + +} // namespace Renderer +} // namespace pulsar diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.backward_dbg.device.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.backward_dbg.device.h new file mode 100644 index 0000000000000000000000000000000000000000..f2bdc7e69027d29a8442a14b08d677cc22dc51c9 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.backward_dbg.device.h @@ -0,0 +1,157 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_RENDERER_BACKWARD_DBG_DEVICE_H_ +#define PULSAR_NATIVE_RENDERER_BACKWARD_DBG_DEVICE_H_ + +#include "./camera.device.h" +#include "./math.h" +#include "./renderer.h" + +namespace pulsar { +namespace Renderer { + +template +void backward_dbg( + Renderer* self, + const float* grad_im, + const float* image, + const float* forw_info, + const float* vert_pos, + const float* vert_col, + const float* vert_rad, + const CamInfo& cam, + const float& gamma, + float percent_allowed_difference, + const uint& max_n_hits, + const float* vert_opy_d, + const size_t& num_balls, + const uint& mode, + const bool& dif_pos, + const bool& dif_col, + const bool& dif_rad, + const bool& dif_cam, + const bool& dif_opy, + const uint& pos_x, + const uint& pos_y, + cudaStream_t stream) { + ARGCHECK(gamma > 0.f && gamma <= 1.f, 6, "gamma must be in [0., 1.]"); + ARGCHECK( + percent_allowed_difference >= 0.f && percent_allowed_difference <= 1.f, + 7, + "percent_allowed_difference must be in [0., 1.]"); + ARGCHECK(max_n_hits >= 1u, 8, "max_n_hits must be >= 1"); + ARGCHECK( + num_balls > 0 && num_balls <= self->max_num_balls, + 9, + "num_balls must be >0 and less than max num balls!"); + ARGCHECK( + cam.film_width == self->cam.film_width && + cam.film_height == self->cam.film_height, + 5, + "cam film size must agree"); + ARGCHECK(mode <= 1, 10, "mode must be <= 1!"); + if (percent_allowed_difference < EPS) { + LOG(WARNING) << "percent_allowed_difference < " << FEPS << "! Clamping to " + << FEPS << "."; + percent_allowed_difference = FEPS; + } + ARGCHECK( + pos_x < cam.film_width && pos_y < cam.film_height, + 15, + "pos_x must be < width and pos_y < height."); + if (percent_allowed_difference > 1.f - FEPS) { + LOG(WARNING) << "percent_allowed_difference > " << (1.f - FEPS) + << "! Clamping to " << (1.f - FEPS) << "."; + percent_allowed_difference = 1.f - FEPS; + } + LOG_IF(INFO, PULSAR_LOG_RENDER) + << "Rendering debug backward pass for x: " << pos_x << ", y: " << pos_y; + // Update camera. + self->cam.eye = cam.eye; + self->cam.pixel_0_0_center = cam.pixel_0_0_center - cam.eye; + self->cam.pixel_dir_x = cam.pixel_dir_x; + self->cam.pixel_dir_y = cam.pixel_dir_y; + self->cam.sensor_dir_z = cam.sensor_dir_z; + self->cam.half_pixel_size = cam.half_pixel_size; + self->cam.focal_length = cam.focal_length; + self->cam.aperture_width = cam.aperture_width; + self->cam.aperture_height = cam.aperture_height; + self->cam.min_dist = cam.min_dist; + self->cam.max_dist = cam.max_dist; + self->cam.norm_fac = cam.norm_fac; + self->cam.principal_point_offset_x = cam.principal_point_offset_x; + self->cam.principal_point_offset_y = cam.principal_point_offset_y; + self->cam.film_border_left = cam.film_border_left; + self->cam.film_border_top = cam.film_border_top; + LAUNCH_MAX_PARALLEL_1D( + calc_signature, + num_balls, + stream, + *self, + reinterpret_cast(vert_pos), + vert_col, + vert_rad, + num_balls); + CHECKLAUNCH(); + MEMSET(self->grad_pos_d, 0, float3, num_balls, stream); + MEMSET(self->grad_col_d, 0, float, num_balls * self->cam.n_channels, stream); + MEMSET(self->grad_rad_d, 0, float, num_balls, stream); + MEMSET(self->grad_cam_d, 0, float, 12, stream); + MEMSET(self->grad_cam_buf_d, 0, CamGradInfo, num_balls, stream); + MEMSET(self->grad_opy_d, 0, float, num_balls, stream); + MEMSET(self->ids_sorted_d, 0, int, num_balls, stream); + LAUNCH_MAX_PARALLEL_2D( + calc_gradients, + (int64_t)1, + (int64_t)1, + stream, + self->cam, + grad_im, + gamma, + reinterpret_cast(vert_pos), + vert_col, + vert_rad, + vert_opy_d, + num_balls, + image, + forw_info, + self->di_d, + self->ii_d, + dif_pos, + dif_col, + dif_rad, + dif_cam, + dif_opy, + self->grad_rad_d, + self->grad_col_d, + self->grad_pos_d, + self->grad_cam_buf_d, + self->grad_opy_d, + self->ids_sorted_d, + self->n_track, + pos_x, + pos_y); + CHECKLAUNCH(); + // We're not doing sphere gradient normalization here. + SUM_WS( + self->grad_cam_buf_d, + reinterpret_cast(self->grad_cam_d), + static_cast(1), + self->workspace_d, + self->workspace_size, + stream); + CHECKLAUNCH(); + // We're not doing camera gradient normalization here. + LOG_IF(INFO, PULSAR_LOG_RENDER) << "Debug backward pass complete."; +} + +} // namespace Renderer +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.backward_dbg.instantiate.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.backward_dbg.instantiate.h new file mode 100644 index 0000000000000000000000000000000000000000..5a7a1ba1f8e56df0a5ff212e7eb769a0564e7f60 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.backward_dbg.instantiate.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "./renderer.backward_dbg.device.h" + +namespace pulsar { +namespace Renderer { + +template void backward_dbg( + Renderer* self, + const float* grad_im, + const float* image, + const float* forw_info, + const float* vert_pos, + const float* vert_col, + const float* vert_rad, + const CamInfo& cam, + const float& gamma, + float percent_allowed_difference, + const uint& max_n_hits, + const float* vert_opy, + const size_t& num_balls, + const uint& mode, + const bool& dif_pos, + const bool& dif_col, + const bool& dif_rad, + const bool& dif_cam, + const bool& dif_opy, + const uint& pos_x, + const uint& pos_y, + cudaStream_t stream); + +} // namespace Renderer +} // namespace pulsar diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.calc_gradients.device.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.calc_gradients.device.h new file mode 100644 index 0000000000000000000000000000000000000000..90b3872e9606c8830b039f18c4d465c3f8c23c1f --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.calc_gradients.device.h @@ -0,0 +1,198 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_INCLUDE_RENDERER_CALC_GRADIENTS_H_ +#define PULSAR_NATIVE_INCLUDE_RENDERER_CALC_GRADIENTS_H_ + +#include "../global.h" +#include "./commands.h" +#include "./renderer.h" + +#include "./renderer.draw.device.h" + +namespace pulsar { +namespace Renderer { + +template +GLOBAL void calc_gradients( + const CamInfo cam, /** Camera in world coordinates. */ + float const* const RESTRICT grad_im, /** The gradient image. */ + const float + gamma, /** The transparency parameter used in the forward pass. */ + float3 const* const RESTRICT vert_poss, /** Vertex position vector. */ + float const* const RESTRICT vert_cols, /** Vertex color vector. */ + float const* const RESTRICT vert_rads, /** Vertex radius vector. */ + float const* const RESTRICT opacity, /** Vertex opacity. */ + const uint num_balls, /** Number of balls. */ + float const* const RESTRICT result_d, /** Result image. */ + float const* const RESTRICT forw_info_d, /** Forward pass info. */ + DrawInfo const* const RESTRICT di_d, /** Draw information. */ + IntersectInfo const* const RESTRICT ii_d, /** Intersect information. */ + // Mode switches. + const bool calc_grad_pos, + const bool calc_grad_col, + const bool calc_grad_rad, + const bool calc_grad_cam, + const bool calc_grad_opy, + // Out variables. + float* const RESTRICT grad_rad_d, /** Radius gradients. */ + float* const RESTRICT grad_col_d, /** Color gradients. */ + float3* const RESTRICT grad_pos_d, /** Position gradients. */ + CamGradInfo* const RESTRICT grad_cam_buf_d, /** Camera gradient buffer. */ + float* const RESTRICT grad_opy_d, /** Opacity gradient buffer. */ + int* const RESTRICT + grad_contributed_d, /** Gradient contribution counter. */ + // Infrastructure. + const int n_track, + const uint offs_x, + const uint offs_y /** Debug offsets. */ +) { + uint limit_x = cam.film_width, limit_y = cam.film_height; + if (offs_x != 0) { + // We're in debug mode. + limit_x = 1; + limit_y = 1; + } + GET_PARALLEL_IDS_2D(coord_x_base, coord_y_base, limit_x, limit_y); + // coord_x_base and coord_y_base are in the film coordinate system. + // We now need to translate to the aperture coordinate system. If + // the principal point was shifted left/up nothing has to be + // subtracted - only shift needs to be added in case it has been + // shifted down/right. + const uint film_coord_x = coord_x_base + offs_x; + const uint ap_coord_x = film_coord_x + + 2 * static_cast(std::max(0, cam.principal_point_offset_x)); + const uint film_coord_y = coord_y_base + offs_y; + const uint ap_coord_y = film_coord_y + + 2 * static_cast(std::max(0, cam.principal_point_offset_y)); + const float3 ray_dir = /** Ray cast through the pixel, normalized. */ + cam.pixel_0_0_center + ap_coord_x * cam.pixel_dir_x + + ap_coord_y * cam.pixel_dir_y; + const float norm_ray_dir = length(ray_dir); + // ray_dir_norm *must* be calculated here in the same way as in the draw + // function to have the same values withno other numerical instabilities + // (for example, ray_dir * FRCP(norm_ray_dir) does not work)! + float3 ray_dir_norm; /** Ray cast through the pixel, normalized. */ + float2 projected_ray; /** Ray intersection with the sensor. */ + if (cam.orthogonal_projection) { + ray_dir_norm = cam.sensor_dir_z; + projected_ray.x = static_cast(ap_coord_x); + projected_ray.y = static_cast(ap_coord_y); + } else { + ray_dir_norm = normalize( + cam.pixel_0_0_center + ap_coord_x * cam.pixel_dir_x + + ap_coord_y * cam.pixel_dir_y); + // This is a reasonable assumption for normal focal lengths and image sizes. + PASSERT(FABS(ray_dir_norm.z) > FEPS); + projected_ray.x = ray_dir_norm.x / ray_dir_norm.z * cam.focal_length; + projected_ray.y = ray_dir_norm.y / ray_dir_norm.z * cam.focal_length; + } + float* result = const_cast( + result_d + film_coord_y * cam.film_width * cam.n_channels + + film_coord_x * cam.n_channels); + const float* grad_im_l = grad_im + + film_coord_y * cam.film_width * cam.n_channels + + film_coord_x * cam.n_channels; + // For writing... + float3 grad_pos; + float grad_rad, grad_opy; + CamGradInfo grad_cam_local = CamGradInfo(); + // Set up shared infrastructure. + const int fwi_loc = film_coord_y * cam.film_width * (3 + 2 * n_track) + + film_coord_x * (3 + 2 * n_track); + float sm_m = forw_info_d[fwi_loc]; + float sm_d = forw_info_d[fwi_loc + 1]; + PULSAR_LOG_DEV_APIX( + PULSAR_LOG_GRAD, + "grad|sm_m: %f, sm_d: %f, result: " + "%f, %f, %f; grad_im: %f, %f, %f.\n", + sm_m, + sm_d, + result[0], + result[1], + result[2], + grad_im_l[0], + grad_im_l[1], + grad_im_l[2]); + // Start processing. + for (int grad_idx = 0; grad_idx < n_track; ++grad_idx) { + int sphere_idx; + FASI(forw_info_d[fwi_loc + 3 + 2 * grad_idx], sphere_idx); + PASSERT( + sphere_idx == -1 || + sphere_idx >= 0 && static_cast(sphere_idx) < num_balls); + if (sphere_idx >= 0) { + // TODO: make more efficient. + grad_pos = make_float3(0.f, 0.f, 0.f); + grad_rad = 0.f; + grad_cam_local = CamGradInfo(); + const DrawInfo di = di_d[sphere_idx]; + grad_opy = 0.f; + draw( + di, + opacity == NULL ? 1.f : opacity[sphere_idx], + cam, + gamma, + ray_dir_norm, + projected_ray, + // Mode switches. + false, // draw only + calc_grad_pos, + calc_grad_col, + calc_grad_rad, + calc_grad_cam, + calc_grad_opy, + // Position info. + ap_coord_x, + ap_coord_y, + sphere_idx, + // Optional in. + &ii_d[sphere_idx], + &ray_dir, + &norm_ray_dir, + grad_im_l, + NULL, + // In/out + &sm_d, + &sm_m, + result, + // Optional out. + NULL, + NULL, + &grad_pos, + grad_col_d + sphere_idx * cam.n_channels, + &grad_rad, + &grad_cam_local, + &grad_opy); + ATOMICADD(&(grad_rad_d[sphere_idx]), grad_rad); + // Color has been added directly. + ATOMICADD_F3(&(grad_pos_d[sphere_idx]), grad_pos); + ATOMICADD_F3( + &(grad_cam_buf_d[sphere_idx].cam_pos), grad_cam_local.cam_pos); + if (!cam.orthogonal_projection) { + ATOMICADD_F3( + &(grad_cam_buf_d[sphere_idx].pixel_0_0_center), + grad_cam_local.pixel_0_0_center); + } + ATOMICADD_F3( + &(grad_cam_buf_d[sphere_idx].pixel_dir_x), + grad_cam_local.pixel_dir_x); + ATOMICADD_F3( + &(grad_cam_buf_d[sphere_idx].pixel_dir_y), + grad_cam_local.pixel_dir_y); + ATOMICADD(&(grad_opy_d[sphere_idx]), grad_opy); + ATOMICADD(&(grad_contributed_d[sphere_idx]), 1); + } + } + END_PARALLEL_2D_NORET(); +}; + +} // namespace Renderer +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.calc_gradients.instantiate.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.calc_gradients.instantiate.h new file mode 100644 index 0000000000000000000000000000000000000000..596c322b28eef850d5466037770cef53caf51cff --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.calc_gradients.instantiate.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "./renderer.calc_gradients.device.h" + +namespace pulsar { +namespace Renderer { + +template GLOBAL void calc_gradients( + const CamInfo cam, /** Camera in world coordinates. */ + float const* const RESTRICT grad_im, /** The gradient image. */ + const float + gamma, /** The transparency parameter used in the forward pass. */ + float3 const* const RESTRICT vert_poss, /** Vertex position vector. */ + float const* const RESTRICT vert_cols, /** Vertex color vector. */ + float const* const RESTRICT vert_rads, /** Vertex radius vector. */ + float const* const RESTRICT opacity, /** Vertex opacity. */ + const uint num_balls, /** Number of balls. */ + float const* const RESTRICT result_d, /** Result image. */ + float const* const RESTRICT forw_info_d, /** Forward pass info. */ + DrawInfo const* const RESTRICT di_d, /** Draw information. */ + IntersectInfo const* const RESTRICT ii_d, /** Intersect information. */ + // Mode switches. + const bool calc_grad_pos, + const bool calc_grad_col, + const bool calc_grad_rad, + const bool calc_grad_cam, + const bool calc_grad_opy, + // Out variables. + float* const RESTRICT grad_rad_d, /** Radius gradients. */ + float* const RESTRICT grad_col_d, /** Color gradients. */ + float3* const RESTRICT grad_pos_d, /** Position gradients. */ + CamGradInfo* const RESTRICT grad_cam_buf_d, /** Camera gradient buffer. */ + float* const RESTRICT grad_opy_d, /** Opacity gradient buffer. */ + int* const RESTRICT + grad_contributed_d, /** Gradient contribution counter. */ + // Infrastructure. + const int n_track, + const uint offs_x, + const uint offs_y); + +} // namespace Renderer +} // namespace pulsar diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.calc_signature.device.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.calc_signature.device.h new file mode 100644 index 0000000000000000000000000000000000000000..bd687fee63d1ee9869ab5beb454a910ff387914c --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.calc_signature.device.h @@ -0,0 +1,201 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_INCLUDE_RENDERER_CALC_SIGNATURE_DEVICE_H_ +#define PULSAR_NATIVE_INCLUDE_RENDERER_CALC_SIGNATURE_DEVICE_H_ + +#include "../global.h" +#include "./camera.device.h" +#include "./commands.h" +#include "./math.h" +#include "./renderer.get_screen_area.device.h" +#include "./renderer.h" + +namespace pulsar { +namespace Renderer { + +template +GLOBAL void calc_signature( + Renderer renderer, + float3 const* const RESTRICT vert_poss, + float const* const RESTRICT vert_cols, + float const* const RESTRICT vert_rads, + const uint num_balls) { + /* We're not using RESTRICT here for the pointers within `renderer`. Just one + value is being read from each of the pointers, so the effect would be + negligible or non-existent. */ + GET_PARALLEL_IDX_1D(idx, num_balls); + // Create aliases. + // For reading... + const float3& vert_pos = vert_poss[idx]; /** Vertex position. */ + const float* vert_col = + vert_cols + idx * renderer.cam.n_channels; /** Vertex color. */ + const float& vert_rad = vert_rads[idx]; /** Vertex radius. */ + const CamInfo& cam = renderer.cam; /** Camera in world coordinates. */ + // For writing... + /** Ball ID (either original index of the ball or -1 if not visible). */ + int& id_out = renderer.ids_d[idx]; + /** Intersection helper structure for the ball. */ + IntersectInfo& intersect_helper_out = renderer.ii_d[idx]; + /** Draw helper structure for this ball. */ + DrawInfo& draw_helper_out = renderer.di_d[idx]; + /** Minimum possible intersection depth for this ball. */ + float& closest_possible_intersect_out = renderer.min_depth_d[idx]; + PULSAR_LOG_DEV( + PULSAR_LOG_CALC_SIGNATURE, + "signature %d|vert_pos: %.9f, %.9f, %.9f, vert_col (first three): " + "%.9f, %.9f, %.9f.\n", + idx, + vert_pos.x, + vert_pos.y, + vert_pos.z, + vert_col[0], + vert_col[1], + vert_col[2]); + // Set flags to invalid for a potential early return. + id_out = -1; // Invalid ID. + closest_possible_intersect_out = + MAX_FLOAT; // These spheres are sorted to the very end. + intersect_helper_out.max.x = MAX_USHORT; // No intersection possible. + intersect_helper_out.min.x = MAX_USHORT; + intersect_helper_out.max.y = MAX_USHORT; + intersect_helper_out.min.y = MAX_USHORT; + // Start processing. + /** Ball center in the camera coordinate system. */ + const float3 ball_center_cam = vert_pos - cam.eye; + /** Distance to the ball center in the camera coordinate system. */ + const float t_center = length(ball_center_cam); + /** Closest possible intersection with this ball from the camera. */ + float closest_possible_intersect; + if (cam.orthogonal_projection) { + const float3 ball_center_cam_rot = rotate( + ball_center_cam, + cam.pixel_dir_x / length(cam.pixel_dir_x), + cam.pixel_dir_y / length(cam.pixel_dir_y), + cam.sensor_dir_z); + closest_possible_intersect = ball_center_cam_rot.z - vert_rad; + } else { + closest_possible_intersect = t_center - vert_rad; + } + PULSAR_LOG_DEV( + PULSAR_LOG_CALC_SIGNATURE, + "signature %d|t_center: %f. vert_rad: %f. " + "closest_possible_intersect: %f.\n", + idx, + t_center, + vert_rad, + closest_possible_intersect); + /** + * Corner points of the enclosing projected rectangle of the ball. + * They are first calculated in the camera coordinate system, then + * converted to the pixel coordinate system. + */ + float x_1, x_2, y_1, y_2; + bool hits_screen_plane; + float3 ray_center_norm = ball_center_cam / t_center; + PASSERT(vert_rad >= 0.f); + if (closest_possible_intersect < cam.min_dist || + closest_possible_intersect > cam.max_dist) { + PULSAR_LOG_DEV( + PULSAR_LOG_CALC_SIGNATURE, + "signature %d|ignoring sphere out of min/max bounds: %.9f, " + "min: %.9f, max: %.9f.\n", + idx, + closest_possible_intersect, + cam.min_dist, + cam.max_dist); + RETURN_PARALLEL(); + } + // Find the relevant region on the screen plane. + hits_screen_plane = get_screen_area( + ball_center_cam, + ray_center_norm, + vert_rad, + cam, + idx, + &x_1, + &x_2, + &y_1, + &y_2); + if (!hits_screen_plane) + RETURN_PARALLEL(); + PULSAR_LOG_DEV( + PULSAR_LOG_CALC_SIGNATURE, + "signature %d|in pixels: x_1: %f, x_2: %f, y_1: %f, y_2: %f.\n", + idx, + x_1, + x_2, + y_1, + y_2); + // Check whether the pixel coordinates are on screen. + if (FMAX(x_1, x_2) <= static_cast(cam.film_border_left) || + FMIN(x_1, x_2) >= + static_cast(cam.film_border_left + cam.film_width) - 0.5f || + FMAX(y_1, y_2) <= static_cast(cam.film_border_top) || + FMIN(y_1, y_2) > + static_cast(cam.film_border_top + cam.film_height) - 0.5f) + RETURN_PARALLEL(); + // Write results. + id_out = idx; + intersect_helper_out.min.x = static_cast( + FMAX(FMIN(x_1, x_2), static_cast(cam.film_border_left))); + intersect_helper_out.min.y = static_cast( + FMAX(FMIN(y_1, y_2), static_cast(cam.film_border_top))); + // In the following calculations, the max that needs to be stored is + // exclusive. + // That means that the calculated value needs to be `ceil`ed and incremented + // to find the correct value. + intersect_helper_out.max.x = static_cast(FMIN( + FCEIL(FMAX(x_1, x_2)) + 1, + static_cast(cam.film_border_left + cam.film_width))); + intersect_helper_out.max.y = static_cast(FMIN( + FCEIL(FMAX(y_1, y_2)) + 1, + static_cast(cam.film_border_top + cam.film_height))); + PULSAR_LOG_DEV( + PULSAR_LOG_CALC_SIGNATURE, + "signature %d|limits after refining: x_1: %u, x_2: %u, " + "y_1: %u, y_2: %u.\n", + idx, + intersect_helper_out.min.x, + intersect_helper_out.max.x, + intersect_helper_out.min.y, + intersect_helper_out.max.y); + if (intersect_helper_out.min.x == MAX_USHORT) { + id_out = -1; + RETURN_PARALLEL(); + } + PULSAR_LOG_DEV( + PULSAR_LOG_CALC_SIGNATURE, + "signature %d|writing info. closest_possible_intersect: %.9f. " + "ray_center_norm: %.9f, %.9f, %.9f. t_center: %.9f. radius: %.9f.\n", + idx, + closest_possible_intersect, + ray_center_norm.x, + ray_center_norm.y, + ray_center_norm.z, + t_center, + vert_rad); + closest_possible_intersect_out = closest_possible_intersect; + draw_helper_out.ray_center_norm = ray_center_norm; + draw_helper_out.t_center = t_center; + draw_helper_out.radius = vert_rad; + if (cam.n_channels <= 3) { + draw_helper_out.first_color = vert_col[0]; + for (uint c_id = 1; c_id < cam.n_channels; ++c_id) { + draw_helper_out.color_union.color[c_id - 1] = vert_col[c_id]; + } + } else { + draw_helper_out.color_union.ptr = const_cast(vert_col); + } + END_PARALLEL(); +}; + +} // namespace Renderer +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.calc_signature.instantiate.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.calc_signature.instantiate.h new file mode 100644 index 0000000000000000000000000000000000000000..6afa95b44b161d8881b79b22e119c89aad522cc6 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.calc_signature.instantiate.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_INCLUDE_RENDERER_CALC_SIGNATURE_INSTANTIATE_H_ +#define PULSAR_NATIVE_INCLUDE_RENDERER_CALC_SIGNATURE_INSTANTIATE_H_ + +#include "./renderer.calc_signature.device.h" + +namespace pulsar { +namespace Renderer { +template GLOBAL void calc_signature( + Renderer renderer, + float3 const* const RESTRICT vert_poss, + float const* const RESTRICT vert_cols, + float const* const RESTRICT vert_rads, + const uint num_balls); +} +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.construct.device.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.construct.device.h new file mode 100644 index 0000000000000000000000000000000000000000..984f7710ba50e4dbeea8e61a2bac7ab41e608697 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.construct.device.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_INCLUDE_RENDERER_CONSTRUCT_DEVICE_H_ +#define PULSAR_NATIVE_INCLUDE_RENDERER_CONSTRUCT_DEVICE_H_ + +#include "../global.h" +#include "./camera.device.h" +#include "./commands.h" +#include "./math.h" +#include "./renderer.h" + +namespace pulsar { +namespace Renderer { + +template +HOST void construct( + Renderer* self, + const size_t& max_num_balls, + const int& width, + const int& height, + const bool& orthogonal_projection, + const bool& right_handed_system, + const float& background_normalization_depth, + const uint& n_channels, + const uint& n_track) { + ARGCHECK( + (max_num_balls > 0 && max_num_balls < MAX_INT), + 2, + ("the maximum number of balls must be >0 and <" + + std::to_string(MAX_INT) + ". Is " + std::to_string(max_num_balls) + ".") + .c_str()); + ARGCHECK(width > 1, 3, "the image width must be > 1"); + ARGCHECK(height > 1, 4, "the image height must be > 1"); + ARGCHECK( + background_normalization_depth > 0.f && + background_normalization_depth < 1.f, + 6, + "background_normalization_depth must be in ]0., 1.[."); + ARGCHECK(n_channels > 0, 7, "n_channels must be >0!"); + ARGCHECK( + n_track > 0 && n_track <= MAX_GRAD_SPHERES, + 8, + ("n_track must be >0 and <" + std::to_string(MAX_GRAD_SPHERES) + ". Is " + + std::to_string(n_track) + ".") + .c_str()); + self->cam.film_width = width; + self->cam.film_height = height; + self->max_num_balls = max_num_balls; + MALLOC(self->result_d, float, width* height* n_channels); + self->cam.orthogonal_projection = orthogonal_projection; + self->cam.right_handed = right_handed_system; + self->cam.background_normalization_depth = background_normalization_depth; + self->cam.n_channels = n_channels; + MALLOC(self->min_depth_d, float, max_num_balls); + MALLOC(self->min_depth_sorted_d, float, max_num_balls); + MALLOC(self->ii_d, IntersectInfo, max_num_balls); + MALLOC(self->ii_sorted_d, IntersectInfo, max_num_balls); + MALLOC(self->ids_d, int, max_num_balls); + MALLOC(self->ids_sorted_d, int, max_num_balls); + size_t sort_id_size = 0; + GET_SORT_WS_SIZE(&sort_id_size, float, int, max_num_balls); + CHECKLAUNCH(); + size_t sort_ii_size = 0; + GET_SORT_WS_SIZE(&sort_ii_size, float, IntersectInfo, max_num_balls); + CHECKLAUNCH(); + size_t sort_di_size = 0; + GET_SORT_WS_SIZE(&sort_di_size, float, DrawInfo, max_num_balls); + CHECKLAUNCH(); + size_t select_ii_size = 0; + GET_SELECT_WS_SIZE(&select_ii_size, char, IntersectInfo, max_num_balls); + size_t select_di_size = 0; + GET_SELECT_WS_SIZE(&select_di_size, char, DrawInfo, max_num_balls); + size_t sum_size = 0; + GET_SUM_WS_SIZE(&sum_size, CamGradInfo, max_num_balls); + size_t sum_cont_size = 0; + GET_SUM_WS_SIZE(&sum_cont_size, int, max_num_balls); + size_t reduce_size = 0; + GET_REDUCE_WS_SIZE( + &reduce_size, IntersectInfo, IntersectInfoMinMax(), max_num_balls); + self->workspace_size = IMAX( + IMAX(IMAX(sort_id_size, sort_ii_size), sort_di_size), + IMAX( + IMAX(select_di_size, select_ii_size), + IMAX(IMAX(sum_size, sum_cont_size), reduce_size))); + MALLOC(self->workspace_d, char, self->workspace_size); + MALLOC(self->di_d, DrawInfo, max_num_balls); + MALLOC(self->di_sorted_d, DrawInfo, max_num_balls); + MALLOC(self->region_flags_d, char, max_num_balls); + MALLOC(self->num_selected_d, size_t, 1); + MALLOC(self->forw_info_d, float, width* height * (3 + 2 * n_track)); + MALLOC(self->min_max_pixels_d, IntersectInfo, 1); + MALLOC(self->grad_pos_d, float3, max_num_balls); + MALLOC(self->grad_col_d, float, max_num_balls* n_channels); + MALLOC(self->grad_rad_d, float, max_num_balls); + MALLOC(self->grad_cam_d, float, 12); + MALLOC(self->grad_cam_buf_d, CamGradInfo, max_num_balls); + MALLOC(self->grad_opy_d, float, max_num_balls); + MALLOC(self->n_grad_contributions_d, int, 1); + self->n_track = static_cast(n_track); +} + +} // namespace Renderer +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.construct.instantiate.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.construct.instantiate.h new file mode 100644 index 0000000000000000000000000000000000000000..e5ce722e29b063b04cb8efc0e880d9332bd35f23 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.construct.instantiate.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_INCLUDE_RENDERER_CONSTRUCT_INSTANTIATE_H_ +#define PULSAR_NATIVE_INCLUDE_RENDERER_CONSTRUCT_INSTANTIATE_H_ + +#include "./renderer.construct.device.h" + +namespace pulsar { +namespace Renderer { +template void construct( + Renderer* self, + const size_t& max_num_balls, + const int& width, + const int& height, + const bool& orthogonal_projection, + const bool& right_handed_system, + const float& background_normalization_depth, + const uint& n_channels, + const uint& n_track); +} +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.create_selector.device.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.create_selector.device.h new file mode 100644 index 0000000000000000000000000000000000000000..747ad03cd3a3a49c34d81485a1780d81a332a215 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.create_selector.device.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_INCLUDE_RENDERER_CREATE_SELECTOR_DEVICE_H_ +#define PULSAR_NATIVE_INCLUDE_RENDERER_CREATE_SELECTOR_DEVICE_H_ + +#include "../global.h" +#include "./commands.h" +#include "./renderer.h" + +namespace pulsar { +namespace Renderer { + +template +GLOBAL void create_selector( + IntersectInfo const* const RESTRICT ii_sorted_d, + const uint num_balls, + const int min_x, + const int max_x, + const int min_y, + const int max_y, + /* Out variables. */ + char* RESTRICT region_flags_d) { + GET_PARALLEL_IDX_1D(idx, num_balls); + bool hit = (static_cast(ii_sorted_d[idx].min.x) <= max_x) && + (static_cast(ii_sorted_d[idx].max.x) > min_x) && + (static_cast(ii_sorted_d[idx].min.y) <= max_y) && + (static_cast(ii_sorted_d[idx].max.y) > min_y); + region_flags_d[idx] = hit; + END_PARALLEL_NORET(); +} + +} // namespace Renderer +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.create_selector.instantiate.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.create_selector.instantiate.h new file mode 100644 index 0000000000000000000000000000000000000000..8e91a8bfb8e9b0f03db39c001e9363920b2eb35f --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.create_selector.instantiate.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_INCLUDE_RENDERER_CREATE_SELECTOR_INSTANTIATE_H_ +#define PULSAR_NATIVE_INCLUDE_RENDERER_CREATE_SELECTOR_INSTANTIATE_H_ + +#include "./renderer.create_selector.device.h" + +namespace pulsar { +namespace Renderer { + +template GLOBAL void create_selector( + IntersectInfo const* const RESTRICT ii_sorted_d, + const uint num_balls, + const int min_x, + const int max_x, + const int min_y, + const int max_y, + /* Out variables. */ + char* RESTRICT region_flags_d); + +} +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.destruct.device.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.destruct.device.h new file mode 100644 index 0000000000000000000000000000000000000000..8520233c59be062fa72376158a9935afa50c3950 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.destruct.device.h @@ -0,0 +1,89 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_INCLUDE_RENDERER_DESTRUCT_H_ +#define PULSAR_NATIVE_INCLUDE_RENDERER_DESTRUCT_H_ + +#include "../global.h" +#include "./commands.h" +#include "./renderer.h" + +namespace pulsar { +namespace Renderer { + +template +HOST void destruct(Renderer* self) { + if (self->result_d != NULL) + FREE(self->result_d); + self->result_d = NULL; + if (self->min_depth_d != NULL) + FREE(self->min_depth_d); + self->min_depth_d = NULL; + if (self->min_depth_sorted_d != NULL) + FREE(self->min_depth_sorted_d); + self->min_depth_sorted_d = NULL; + if (self->ii_d != NULL) + FREE(self->ii_d); + self->ii_d = NULL; + if (self->ii_sorted_d != NULL) + FREE(self->ii_sorted_d); + self->ii_sorted_d = NULL; + if (self->ids_d != NULL) + FREE(self->ids_d); + self->ids_d = NULL; + if (self->ids_sorted_d != NULL) + FREE(self->ids_sorted_d); + self->ids_sorted_d = NULL; + if (self->workspace_d != NULL) + FREE(self->workspace_d); + self->workspace_d = NULL; + if (self->di_d != NULL) + FREE(self->di_d); + self->di_d = NULL; + if (self->di_sorted_d != NULL) + FREE(self->di_sorted_d); + self->di_sorted_d = NULL; + if (self->region_flags_d != NULL) + FREE(self->region_flags_d); + self->region_flags_d = NULL; + if (self->num_selected_d != NULL) + FREE(self->num_selected_d); + self->num_selected_d = NULL; + if (self->forw_info_d != NULL) + FREE(self->forw_info_d); + self->forw_info_d = NULL; + if (self->min_max_pixels_d != NULL) + FREE(self->min_max_pixels_d); + self->min_max_pixels_d = NULL; + if (self->grad_pos_d != NULL) + FREE(self->grad_pos_d); + self->grad_pos_d = NULL; + if (self->grad_col_d != NULL) + FREE(self->grad_col_d); + self->grad_col_d = NULL; + if (self->grad_rad_d != NULL) + FREE(self->grad_rad_d); + self->grad_rad_d = NULL; + if (self->grad_cam_d != NULL) + FREE(self->grad_cam_d); + self->grad_cam_d = NULL; + if (self->grad_cam_buf_d != NULL) + FREE(self->grad_cam_buf_d); + self->grad_cam_buf_d = NULL; + if (self->grad_opy_d != NULL) + FREE(self->grad_opy_d); + self->grad_opy_d = NULL; + if (self->n_grad_contributions_d != NULL) + FREE(self->n_grad_contributions_d); + self->n_grad_contributions_d = NULL; +} + +} // namespace Renderer +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.destruct.instantiate.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.destruct.instantiate.h new file mode 100644 index 0000000000000000000000000000000000000000..d41ba5a323d0bed9196dc804fab87929b2a726af --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.destruct.instantiate.h @@ -0,0 +1,20 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_INCLUDE_RENDERER_DESTRUCT_INSTANTIATE_H_ +#define PULSAR_NATIVE_INCLUDE_RENDERER_DESTRUCT_INSTANTIATE_H_ + +#include "./renderer.destruct.device.h" + +namespace pulsar { +namespace Renderer { +template void destruct(Renderer* self); +} +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.draw.device.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.draw.device.h new file mode 100644 index 0000000000000000000000000000000000000000..cb8ecabed3eefce77f7120d234fad15b0bed064c --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.draw.device.h @@ -0,0 +1,846 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_INCLUDE_RENDERER_CALC_SIGNATURE_DEVICE_H_ +#define PULSAR_NATIVE_INCLUDE_RENDERER_CALC_SIGNATURE_DEVICE_H_ + +#include "../global.h" +#include "./camera.device.h" +#include "./commands.h" +#include "./math.h" +#include "./renderer.h" + +namespace pulsar { +namespace Renderer { + +/** + * Draw a ball into the `result`. + * + * Returns whether a hit was noticed. See README for an explanation of sphere + * points and variable notation. + */ +INLINE DEVICE bool draw( + /* In variables. */ + const DrawInfo& draw_info, /** The draw information for this ball. */ + const float& opacity, /** The sphere opacity. */ + const CamInfo& + cam, /** Camera information. Doesn't have to be normalized. */ + const float& gamma, /** 'Transparency' indicator (see paper for details). */ + const float3& ray_dir_norm, /** The direction of the ray, normalized. */ + const float2& projected_ray, /** The intersection of the ray with the image + in pixel space. */ + /** Mode switches. */ + const bool& draw_only, /** Whether we are in draw vs. grad mode. */ + const bool& calc_grad_pos, /** Calculate position gradients. */ + const bool& calc_grad_col, /** Calculate color gradients. */ + const bool& calc_grad_rad, /** Calculate radius gradients. */ + const bool& calc_grad_cam, /** Calculate camera gradients. */ + const bool& calc_grad_opy, /** Calculate opacity gradients. */ + /** Position info. */ + const uint& coord_x, /** The pixel position x to draw at. */ + const uint& coord_y, /** The pixel position y to draw at. */ + const uint& idx, /** The id of the sphere to process. */ + /* Optional in variables. */ + IntersectInfo const* const RESTRICT + intersect_info, /** The intersect information for this ball. */ + float3 const* const RESTRICT ray_dir, /** The ray direction (not normalized) + to draw at. Only used for grad computation. */ + float const* const RESTRICT norm_ray_dir, /** The length of the direction + vector. Only used for grad computation. */ + float const* const RESTRICT grad_pix, /** The gradient for this pixel. Only + used for grad computation. */ + float const* const RESTRICT + ln_pad_over_1minuspad, /** Allowed percentage indicator. */ + /* In or out variables, depending on mode. */ + float* const RESTRICT sm_d, /** Normalization denominator. */ + float* const RESTRICT + sm_m, /** Maximum of normalization weight factors observed. */ + float* const RESTRICT + result, /** Result pixel color. Must be zeros initially. */ + /* Optional out variables. */ + float* const RESTRICT depth_threshold, /** The depth threshold to use. Only + used for rendering. */ + float* const RESTRICT intersection_depth_norm_out, /** The intersection + depth. Only set when rendering. */ + float3* const RESTRICT grad_pos, /** Gradient w.r.t. position. */ + float* const RESTRICT grad_col, /** Gradient w.r.t. color. */ + float* const RESTRICT grad_rad, /** Gradient w.r.t. radius. */ + CamGradInfo* const RESTRICT grad_cam, /** Gradient w.r.t. camera. */ + float* const RESTRICT grad_opy /** Gradient w.r.t. opacity. */ +) { + // TODO: variable reuse? + PASSERT( + isfinite(draw_info.ray_center_norm.x) && + isfinite(draw_info.ray_center_norm.y) && + isfinite(draw_info.ray_center_norm.z)); + PASSERT(isfinite(draw_info.t_center) && draw_info.t_center >= 0.f); + PASSERT( + isfinite(draw_info.radius) && draw_info.radius >= 0.f && + draw_info.radius <= draw_info.t_center); + PASSERT(isfinite(ray_dir_norm.x)); + PASSERT(isfinite(ray_dir_norm.y)); + PASSERT(isfinite(ray_dir_norm.z)); + PASSERT(isfinite(*sm_d)); + PASSERT( + cam.orthogonal_projection && cam.focal_length == 0.f || + cam.focal_length > 0.f); + PASSERT(gamma <= 1.f && gamma >= 1e-5f); + /** The ball center in the camera coordinate system. */ + float3 center = draw_info.ray_center_norm * draw_info.t_center; + /** The vector from the reference point to the ball center. */ + float3 raydiff; + if (cam.orthogonal_projection) { + center = rotate( + center, + cam.pixel_dir_x / length(cam.pixel_dir_x), + cam.pixel_dir_y / length(cam.pixel_dir_y), + cam.sensor_dir_z); + raydiff = + make_float3( // TODO: make offset consistent with `get_screen_area`. + center.x - + (projected_ray.x - + static_cast(cam.aperture_width) * .5f) * + (2.f * cam.half_pixel_size), + center.y - + (projected_ray.y - + static_cast(cam.aperture_height) * .5f) * + (2.f * cam.half_pixel_size), + 0.f); + } else { + /** The reference point on the ray; the point in the same distance + * from the camera as the ball center, but along the ray. + */ + const float3 rayref = ray_dir_norm * draw_info.t_center; + raydiff = center - rayref; + } + /** The closeness of the reference point to ball center in world coords. + * + * In [0., radius]. + */ + const float closeness_world = length(raydiff); + /** The reciprocal radius. */ + const float radius_rcp = FRCP(draw_info.radius); + /** The closeness factor normalized with the ball radius. + * + * In [0., 1.]. + */ + float closeness = FSATURATE(FMA(-closeness_world, radius_rcp, 1.f)); + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_DRAW_PIX, + "drawprep %u|center: %.9f, %.9f, %.9f. raydiff: %.9f, " + "%.9f, %.9f. closeness_world: %.9f. closeness: %.9f\n", + idx, + center.x, + center.y, + center.z, + raydiff.x, + raydiff.y, + raydiff.z, + closeness_world, + closeness); + /** Whether this is the 'center pixel' for this ball, the pixel that + * is closest to its projected center. This information is used to + * make sure to draw 'tiny' spheres with less than one pixel in + * projected size. + */ + bool ray_through_center_pixel; + float projected_radius, projected_x, projected_y; + if (cam.orthogonal_projection) { + projected_x = center.x / (2.f * cam.half_pixel_size) + + (static_cast(cam.aperture_width) - 1.f) / 2.f; + projected_y = center.y / (2.f * cam.half_pixel_size) + + (static_cast(cam.aperture_height) - 1.f) / 2.f; + projected_radius = draw_info.radius / (2.f * cam.half_pixel_size); + ray_through_center_pixel = + (FABS(FSUB(projected_x, projected_ray.x)) < 0.5f + FEPS && + FABS(FSUB(projected_y, projected_ray.y)) < 0.5f + FEPS); + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_DRAW_PIX, + "drawprep %u|closeness_world: %.9f. closeness: %.9f. " + "projected (x, y): %.9f, %.9f. projected_ray (x, y): " + "%.9f, %.9f. ray_through_center_pixel: %d.\n", + idx, + closeness_world, + closeness, + projected_x, + projected_y, + projected_ray.x, + projected_ray.y, + ray_through_center_pixel); + } else { + // Misusing this variable for half pixel size projected to the depth + // at which the sphere resides. Leave some slack for numerical + // inaccuracy (factor 1.5). + projected_x = FMUL(cam.half_pixel_size * 1.5, draw_info.t_center) * + FRCP(cam.focal_length); + projected_radius = FMUL(draw_info.radius, cam.focal_length) * + FRCP(draw_info.t_center) / (2.f * cam.half_pixel_size); + ray_through_center_pixel = projected_x > closeness_world; + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_DRAW_PIX, + "drawprep %u|closeness_world: %.9f. closeness: %.9f. " + "projected half pixel size: %.9f. " + "ray_through_center_pixel: %d.\n", + idx, + closeness_world, + closeness, + projected_x, + ray_through_center_pixel); + } + if (draw_only && draw_info.radius < closeness_world && + !ray_through_center_pixel) { + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_DRAW_PIX, + "drawprep %u|Abandoning since no hit has been detected.\n", + idx); + return false; + } else { + // This is always a hit since we are following the forward execution pass. + // p2 is the closest intersection point with the sphere. + } + if (ray_through_center_pixel && projected_radius < 1.f) { + // Make a tiny sphere visible. + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_DRAW_PIX, + "drawprep %u|Setting closeness to 1 (projected radius: %.9f).\n", + idx, + projected_radius); + closeness = 1.; + } + PASSERT(closeness >= 0.f && closeness <= 1.f); + /** Distance between the camera (`o`) and `p1`, the closest point to the + * ball center along the casted ray. + * + * In [t_center - radius, t_center]. + */ + float o__p1_; + /** The distance from ball center to p1. + * + * In [0., sqrt(t_center ^ 2 - (t_center - radius) ^ 2)]. + */ + float c__p1_; + if (cam.orthogonal_projection) { + o__p1_ = FABS(center.z); + c__p1_ = length(raydiff); + } else { + o__p1_ = dot(center, ray_dir_norm); + /** + * This is being calculated as sqrt(t_center^2 - o__p1_^2) = + * sqrt((t_center + o__p1_) * (t_center - o__p1_)) to avoid + * catastrophic cancellation in floating point representations. + */ + c__p1_ = FSQRT( + (draw_info.t_center + o__p1_) * FMAX(draw_info.t_center - o__p1_, 0.f)); + // PASSERT(o__p1_ >= draw_info.t_center - draw_info.radius); + // Numerical errors lead to too large values. + o__p1_ = FMIN(o__p1_, draw_info.t_center); + // PASSERT(o__p1_ <= draw_info.t_center); + } + /** The distance from the closest point to the sphere center (p1) + * to the closest intersection point (p2). + * + * In [0., radius]. + */ + const float p1__p2_ = + FSQRT((draw_info.radius + c__p1_) * FMAX(draw_info.radius - c__p1_, 0.f)); + PASSERT(p1__p2_ >= 0.f && p1__p2_ <= draw_info.radius); + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_DRAW_PIX, + "drawprep %u|o__p1_: %.9f, c__p1_: %.9f, p1__p2_: %.9f.\n", + idx, + o__p1_, + c__p1_, + p1__p2_); + /** The intersection depth of the ray with this ball. + * + * In [t_center - radius, t_center]. + */ + const float intersection_depth = (o__p1_ - p1__p2_); + PASSERT( + cam.orthogonal_projection && + (intersection_depth >= center.z - draw_info.radius && + intersection_depth <= center.z) || + intersection_depth >= draw_info.t_center - draw_info.radius && + intersection_depth <= draw_info.t_center); + /** Normalized distance of the closest intersection point; in [0., 1.]. */ + const float norm_dist = + FMUL(FSUB(intersection_depth, cam.min_dist), cam.norm_fac); + PASSERT(norm_dist >= 0.f && norm_dist <= 1.f); + /** Scaled, normalized distance in [1., 0.] (closest, farthest). */ + const float norm_dist_scaled = FSUB(1.f, norm_dist) / gamma * opacity; + PASSERT(norm_dist_scaled >= 0.f && norm_dist_scaled <= 1.f / gamma); + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_DRAW_PIX, + "drawprep %u|intersection_depth: %.9f, norm_dist: %.9f, " + "norm_dist_scaled: %.9f.\n", + idx, + intersection_depth, + norm_dist, + norm_dist_scaled); + float const* const col_ptr = + cam.n_channels > 3 ? draw_info.color_union.ptr : &draw_info.first_color; + // The implementation for the numerically stable weighted softmax is based + // on https://arxiv.org/pdf/1805.02867.pdf . + if (draw_only) { + /** The old maximum observed value. */ + const float sm_m_old = *sm_m; + *sm_m = FMAX(*sm_m, norm_dist_scaled); + const float coeff_exp = FEXP(norm_dist_scaled - *sm_m); + PASSERT(isfinite(coeff_exp)); + /** The color coefficient for the ball color; in [0., 1.]. */ + const float coeff = closeness * coeff_exp * opacity; + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_DRAW_PIX, + "draw %u|coeff: %.9f. closeness: %.9f. coeff_exp: %.9f. " + "opacity: %.9f.\n", + idx, + coeff, + closeness, + coeff_exp, + opacity); + // Rendering. + if (sm_m_old == *sm_m) { + // Use the fact that exp(0) = 1 to avoid the exp calculation for + // the case that the maximum remains the same (which it should + // most of the time). + *sm_d = FADD(*sm_d, coeff); + for (uint c_id = 0; c_id < cam.n_channels; ++c_id) { + PASSERT(isfinite(result[c_id])); + result[c_id] = FMA(coeff, col_ptr[c_id], result[c_id]); + } + } else { + const float exp_correction = FEXP(sm_m_old - *sm_m); + *sm_d = FMA(*sm_d, exp_correction, coeff); + for (uint c_id = 0; c_id < cam.n_channels; ++c_id) { + PASSERT(isfinite(result[c_id])); + result[c_id] = + FMA(coeff, col_ptr[c_id], FMUL(result[c_id], exp_correction)); + } + } + PASSERT(isfinite(*sm_d)); + *intersection_depth_norm_out = intersection_depth; + // Update the depth threshold. + *depth_threshold = + 1.f - (FLN(*sm_d + FEPS) + *ln_pad_over_1minuspad + *sm_m) * gamma; + *depth_threshold = + FMA(*depth_threshold, FSUB(cam.max_dist, cam.min_dist), cam.min_dist); + } else { + // Gradient computation. + const float coeff_exp = FEXP(norm_dist_scaled - *sm_m); + const float gamma_rcp = FRCP(gamma); + const float radius_sq = FMUL(draw_info.radius, draw_info.radius); + const float coeff = FMAX( + FMIN(closeness * coeff_exp * opacity, *sm_d - FEPS), + 0.f); // in [0., sm_d - FEPS]. + PASSERT(coeff >= 0.f && coeff <= *sm_d); + const float otherw = *sm_d - coeff; // in [FEPS, sm_d]. + const float p1__p2_safe = FMAX(p1__p2_, FEPS); // in [eps, t_center]. + const float cam_range = FSUB(cam.max_dist, cam.min_dist); // in ]0, inf[ + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|pos: %.9f, %.9f, %.9f. pixeldirx: %.9f, %.9f, %.9f. " + "pixeldiry: %.9f, %.9f, %.9f. pixel00center: %.9f, %.9f, %.9f.\n", + idx, + draw_info.ray_center_norm.x * draw_info.t_center, + draw_info.ray_center_norm.y * draw_info.t_center, + draw_info.ray_center_norm.z * draw_info.t_center, + cam.pixel_dir_x.x, + cam.pixel_dir_x.y, + cam.pixel_dir_x.z, + cam.pixel_dir_y.x, + cam.pixel_dir_y.y, + cam.pixel_dir_y.z, + cam.pixel_0_0_center.x, + cam.pixel_0_0_center.y, + cam.pixel_0_0_center.z); + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|ray_dir: %.9f, %.9f, %.9f. " + "ray_dir_norm: %.9f, %.9f, %.9f. " + "draw_info.ray_center_norm: %.9f, %.9f, %.9f.\n", + idx, + ray_dir->x, + ray_dir->y, + ray_dir->z, + ray_dir_norm.x, + ray_dir_norm.y, + ray_dir_norm.z, + draw_info.ray_center_norm.x, + draw_info.ray_center_norm.y, + draw_info.ray_center_norm.z); + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|coeff_exp: %.9f. " + "norm_dist_scaled: %.9f. cam.norm_fac: %f.\n", + idx, + coeff_exp, + norm_dist_scaled, + cam.norm_fac); + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|p1__p2_: %.9f. p1__p2_safe: %.9f.\n", + idx, + p1__p2_, + p1__p2_safe); + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|o__p1_: %.9f. c__p1_: %.9f.\n", + idx, + o__p1_, + c__p1_); + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|intersection_depth: %f. norm_dist: %f. " + "coeff: %.9f. closeness: %f. coeff_exp: %f. opacity: " + "%f. color: %f, %f, %f.\n", + idx, + intersection_depth, + norm_dist, + coeff, + closeness, + coeff_exp, + opacity, + draw_info.first_color, + draw_info.color_union.color[0], + draw_info.color_union.color[1]); + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|t_center: %.9f. " + "radius: %.9f. max_dist: %f. min_dist: %f. gamma: %f.\n", + idx, + draw_info.t_center, + draw_info.radius, + cam.max_dist, + cam.min_dist, + gamma); + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|sm_d: %f. sm_m: %f. grad_pix (first three): %f, %f, %f.\n", + idx, + *sm_d, + *sm_m, + grad_pix[0], + grad_pix[1], + grad_pix[2]); + PULSAR_LOG_DEV_PIX(PULSAR_LOG_GRAD, "grad %u|otherw: %f.\n", idx, otherw); + if (calc_grad_col) { + const float sm_d_norm = FRCP(FMAX(*sm_d, FEPS)); + // First do the multiplication of coeff (in [0., sm_d]) and 1/sm_d. The + // result is a factor in [0., 1.] to be multiplied with the incoming + // gradient. + for (uint c_id = 0; c_id < cam.n_channels; ++c_id) { + ATOMICADD(grad_col + c_id, grad_pix[c_id] * FMUL(coeff, sm_d_norm)); + } + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dimDdcol.x: %f. dresDdcol.x: %f.\n", + idx, + FMUL(coeff, sm_d_norm) * grad_pix[0], + coeff * sm_d_norm); + } + // We disable the computation for too small spheres. + // The comparison is made this way to avoid subtraction of unsigned types. + if (calc_grad_cam || calc_grad_pos || calc_grad_rad || calc_grad_opy) { + //! First find dimDdcoeff. + const float n0 = + otherw * FRCP(FMAX(*sm_d * *sm_d, FEPS)); // in [0., 1. / sm_d]. + PASSERT(isfinite(n0) && n0 >= 0. && n0 <= 1. / *sm_d + 1e2f * FEPS); + // We'll aggergate dimDdcoeff over all the 'color' channels. + float dimDdcoeff = 0.f; + const float otherw_safe_rcp = FRCP(FMAX(otherw, FEPS)); + float othercol; + for (uint c_id = 0; c_id < cam.n_channels; ++c_id) { + othercol = + (result[c_id] * *sm_d - col_ptr[c_id] * coeff) * otherw_safe_rcp; + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|othercol[%u]: %.9f.\n", + idx, + c_id, + othercol); + dimDdcoeff += + FMUL(FMUL(grad_pix[c_id], FSUB(col_ptr[c_id], othercol)), n0); + } + PASSERT(isfinite(dimDdcoeff)); + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dimDdcoeff: %.9f, n0: %f.\n", + idx, + dimDdcoeff, + n0); + if (calc_grad_opy) { + //! dimDdopacity. + *grad_opy += dimDdcoeff * coeff_exp * closeness * + (1.f + opacity * (1.f - norm_dist) * gamma_rcp); + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dcoeffDdopacity: %.9f, dimDdopacity: %.9f.\n", + idx, + coeff_exp * closeness, + dimDdcoeff * coeff_exp * closeness); + } + if (intersect_info->max.x >= intersect_info->min.x + 3 && + intersect_info->max.y >= intersect_info->min.y + 3) { + //! Now find dcoeffDdintersection_depth and dcoeffDdcloseness. + const float dcoeffDdintersection_depth = + -closeness * coeff_exp * opacity * opacity / (gamma * cam_range); + const float dcoeffDdcloseness = coeff_exp * opacity; + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dcoeffDdintersection_depth: %.9f. " + "dimDdintersection_depth: %.9f. " + "dcoeffDdcloseness: %.9f. dimDdcloseness: %.9f.\n", + idx, + dcoeffDdintersection_depth, + dimDdcoeff * dcoeffDdintersection_depth, + dcoeffDdcloseness, + dimDdcoeff * dcoeffDdcloseness); + //! Here, the execution paths for orthogonal and pinyhole camera split. + if (cam.orthogonal_projection) { + if (calc_grad_rad) { + //! Find dcoeffDdrad. + float dcoeffDdrad = + dcoeffDdcloseness * (closeness_world / radius_sq) - + dcoeffDdintersection_depth * draw_info.radius / p1__p2_safe; + PASSERT(isfinite(dcoeffDdrad)); + *grad_rad += FMUL(dimDdcoeff, dcoeffDdrad); + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dimDdrad: %.9f. dcoeffDdrad: %.9f.\n", + idx, + FMUL(dimDdcoeff, dcoeffDdrad), + dcoeffDdrad); + } + if (calc_grad_pos || calc_grad_cam) { + float3 dimDdcenter = raydiff / + p1__p2_safe; /* making it dintersection_depthDdcenter. */ + dimDdcenter.z = sign_dir(center.z); + PASSERT(FABS(center.z) >= cam.min_dist && cam.min_dist >= FEPS); + dimDdcenter *= dcoeffDdintersection_depth; // dcoeffDdcenter + dimDdcenter -= dcoeffDdcloseness * /* dclosenessDdcenter. */ + raydiff * FRCP(FMAX(length(raydiff) * draw_info.radius, FEPS)); + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dcoeffDdcenter: %.9f, %.9f, %.9f.\n", + idx, + dimDdcenter.x, + dimDdcenter.y, + dimDdcenter.z); + // Now dcoeffDdcenter is stored in dimDdcenter. + dimDdcenter *= dimDdcoeff; + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dimDdcenter: %.9f, %.9f, %.9f.\n", + idx, + dimDdcenter.x, + dimDdcenter.y, + dimDdcenter.z); + // Prepare for posglob and cam pos. + const float pixel_size = length(cam.pixel_dir_x); + // pixel_size is the same as length(pixeldiry)! + const float pixel_size_rcp = FRCP(pixel_size); + float3 dcenterDdposglob = + (cam.pixel_dir_x + cam.pixel_dir_y) * pixel_size_rcp + + cam.sensor_dir_z; + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dcenterDdposglob: %.9f, %.9f, %.9f.\n", + idx, + dcenterDdposglob.x, + dcenterDdposglob.y, + dcenterDdposglob.z); + if (calc_grad_pos) { + //! dcenterDdposglob. + *grad_pos += dimDdcenter * dcenterDdposglob; + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dimDdpos: %.9f, %.9f, %.9f.\n", + idx, + dimDdcenter.x * dcenterDdposglob.x, + dimDdcenter.y * dcenterDdposglob.y, + dimDdcenter.z * dcenterDdposglob.z); + } + if (calc_grad_cam) { + //! Camera. + grad_cam->cam_pos -= dimDdcenter * dcenterDdposglob; + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dimDdeye: %.9f, %.9f, %.9f.\n", + idx, + -dimDdcenter.x * dcenterDdposglob.x, + -dimDdcenter.y * dcenterDdposglob.y, + -dimDdcenter.z * dcenterDdposglob.z); + // coord_world + /* + float3 dclosenessDdcoord_world = + raydiff * FRCP(FMAX(draw_info.radius * length(raydiff), FEPS)); + float3 dintersection_depthDdcoord_world = -2.f * raydiff; + */ + float3 dimDdcoord_world = /* dcoeffDdcoord_world */ + dcoeffDdcloseness * raydiff * + FRCP(FMAX(draw_info.radius * length(raydiff), FEPS)) - + dcoeffDdintersection_depth * raydiff / p1__p2_safe; + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dcoeffDdcoord_world: %.9f, %.9f, %.9f.\n", + idx, + dimDdcoord_world.x, + dimDdcoord_world.y, + dimDdcoord_world.z); + dimDdcoord_world *= dimDdcoeff; + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dimDdcoord_world: %.9f, %.9f, %.9f.\n", + idx, + dimDdcoord_world.x, + dimDdcoord_world.y, + dimDdcoord_world.z); + // The third component of dimDdcoord_world is 0! + PASSERT(dimDdcoord_world.z == 0.f); + float3 coord_world = center - raydiff; + coord_world.z = 0.f; + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|coord_world: %.9f, %.9f, %.9f.\n", + idx, + coord_world.x, + coord_world.y, + coord_world.z); + // Do this component-wise to save unnecessary matmul steps. + grad_cam->pixel_dir_x += dimDdcoord_world.x * cam.pixel_dir_x * + coord_world.x * pixel_size_rcp * pixel_size_rcp; + grad_cam->pixel_dir_x += dimDdcoord_world.y * cam.pixel_dir_x * + coord_world.y * pixel_size_rcp * pixel_size_rcp; + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dimDdpixel_dir_x|coord_world: %.9f, %.9f, %.9f.\n", + idx, + grad_cam->pixel_dir_x.x, + grad_cam->pixel_dir_x.y, + grad_cam->pixel_dir_x.z); + // dcenterkDdpixel_dir_k. + float3 center_in_pixels = draw_info.ray_center_norm * + draw_info.t_center * pixel_size_rcp; + grad_cam->pixel_dir_x += dimDdcenter.x * + (center_in_pixels - + outer_product_sum(cam.pixel_dir_x) * center_in_pixels * + pixel_size_rcp * pixel_size_rcp); + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dcenter0dpixel_dir_x: %.9f, %.9f, %.9f.\n", + idx, + (center_in_pixels - + outer_product_sum(cam.pixel_dir_x) * center_in_pixels * + pixel_size_rcp * pixel_size_rcp) + .x, + (center_in_pixels - + outer_product_sum(cam.pixel_dir_x) * center_in_pixels * + pixel_size_rcp * pixel_size_rcp) + .y, + (center_in_pixels - + outer_product_sum(cam.pixel_dir_x) * center_in_pixels * + pixel_size_rcp * pixel_size_rcp) + .z); + grad_cam->pixel_dir_y += dimDdcenter.y * + (center_in_pixels - + outer_product_sum(cam.pixel_dir_y) * center_in_pixels * + pixel_size_rcp * pixel_size_rcp); + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dcenter1dpixel_dir_y: %.9f, %.9f, %.9f.\n", + idx, + (center_in_pixels - + outer_product_sum(cam.pixel_dir_y) * center_in_pixels * + pixel_size_rcp * pixel_size_rcp) + .x, + (center_in_pixels - + outer_product_sum(cam.pixel_dir_y) * center_in_pixels * + pixel_size_rcp * pixel_size_rcp) + .y, + (center_in_pixels - + outer_product_sum(cam.pixel_dir_y) * center_in_pixels * + pixel_size_rcp * pixel_size_rcp) + .z); + // dcenterzDdpixel_dir_k. + float sensordirz_norm_rcp = FRCP( + FMAX(length(cross(cam.pixel_dir_y, cam.pixel_dir_x)), FEPS)); + grad_cam->pixel_dir_x += dimDdcenter.z * + (dot(center, cam.sensor_dir_z) * + cross(cam.pixel_dir_y, cam.sensor_dir_z) - + cross(cam.pixel_dir_y, center)) * + sensordirz_norm_rcp; + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dcenterzDdpixel_dir_x: %.9f, %.9f, %.9f.\n", + idx, + ((dot(center, cam.sensor_dir_z) * + cross(cam.pixel_dir_y, cam.sensor_dir_z) - + cross(cam.pixel_dir_y, center)) * + sensordirz_norm_rcp) + .x, + ((dot(center, cam.sensor_dir_z) * + cross(cam.pixel_dir_y, cam.sensor_dir_z) - + cross(cam.pixel_dir_y, center)) * + sensordirz_norm_rcp) + .y, + ((dot(center, cam.sensor_dir_z) * + cross(cam.pixel_dir_y, cam.sensor_dir_z) - + cross(cam.pixel_dir_y, center)) * + sensordirz_norm_rcp) + .z); + grad_cam->pixel_dir_y += dimDdcenter.z * + (dot(center, cam.sensor_dir_z) * + cross(cam.pixel_dir_x, cam.sensor_dir_z) - + cross(cam.pixel_dir_x, center)) * + sensordirz_norm_rcp; + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dcenterzDdpixel_dir_y: %.9f, %.9f, %.9f.\n", + idx, + ((dot(center, cam.sensor_dir_z) * + cross(cam.pixel_dir_x, cam.sensor_dir_z) - + cross(cam.pixel_dir_x, center)) * + sensordirz_norm_rcp) + .x, + ((dot(center, cam.sensor_dir_z) * + cross(cam.pixel_dir_x, cam.sensor_dir_z) - + cross(cam.pixel_dir_x, center)) * + sensordirz_norm_rcp) + .y, + ((dot(center, cam.sensor_dir_z) * + cross(cam.pixel_dir_x, cam.sensor_dir_z) - + cross(cam.pixel_dir_x, center)) * + sensordirz_norm_rcp) + .z); + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dimDdpixel_dir_x: %.9f, %.9f, %.9f.\n", + idx, + grad_cam->pixel_dir_x.x, + grad_cam->pixel_dir_x.y, + grad_cam->pixel_dir_x.z); + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dimDdpixel_dir_y: %.9f, %.9f, %.9f.\n", + idx, + grad_cam->pixel_dir_y.x, + grad_cam->pixel_dir_y.y, + grad_cam->pixel_dir_y.z); + } + } + } else { + if (calc_grad_rad) { + //! Find dcoeffDdrad. + float dcoeffDdrad = + dcoeffDdcloseness * (closeness_world / radius_sq) - + dcoeffDdintersection_depth * draw_info.radius / p1__p2_safe; + PASSERT(isfinite(dcoeffDdrad)); + *grad_rad += FMUL(dimDdcoeff, dcoeffDdrad); + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dimDdrad: %.9f. dcoeffDdrad: %.9f.\n", + idx, + FMUL(dimDdcoeff, dcoeffDdrad), + dcoeffDdrad); + } + if (calc_grad_pos || calc_grad_cam) { + const float3 tmp1 = center - ray_dir_norm * o__p1_; + const float3 tmp1n = tmp1 / p1__p2_safe; + const float ray_dir_normDotRaydiff = dot(ray_dir_norm, raydiff); + const float3 dcoeffDdray = dcoeffDdintersection_depth * + (tmp1 - o__p1_ * tmp1n) / *norm_ray_dir + + dcoeffDdcloseness * + (ray_dir_norm * -ray_dir_normDotRaydiff + raydiff) / + (closeness_world * draw_info.radius) * + (draw_info.t_center / *norm_ray_dir); + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dcoeffDdray: %.9f, %.9f, %.9f. dimDdray: " + "%.9f, %.9f, %.9f.\n", + idx, + dcoeffDdray.x, + dcoeffDdray.y, + dcoeffDdray.z, + dimDdcoeff * dcoeffDdray.x, + dimDdcoeff * dcoeffDdray.y, + dimDdcoeff * dcoeffDdray.z); + const float3 dcoeffDdcenter = + dcoeffDdintersection_depth * (ray_dir_norm + tmp1n) + + dcoeffDdcloseness * + (draw_info.ray_center_norm * ray_dir_normDotRaydiff - + raydiff) / + (closeness_world * draw_info.radius); + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dcoeffDdcenter: %.9f, %.9f, %.9f. " + "dimDdcenter: %.9f, %.9f, %.9f.\n", + idx, + dcoeffDdcenter.x, + dcoeffDdcenter.y, + dcoeffDdcenter.z, + dimDdcoeff * dcoeffDdcenter.x, + dimDdcoeff * dcoeffDdcenter.y, + dimDdcoeff * dcoeffDdcenter.z); + if (calc_grad_pos) { + *grad_pos += dimDdcoeff * dcoeffDdcenter; + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dimDdposglob: %.9f, %.9f, %.9f.\n", + idx, + dimDdcoeff * dcoeffDdcenter.x, + dimDdcoeff * dcoeffDdcenter.y, + dimDdcoeff * dcoeffDdcenter.z); + } + if (calc_grad_cam) { + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dimDdeye: %.9f, %.9f, %.9f.\n", + idx, + -dimDdcoeff * (dcoeffDdcenter.x + dcoeffDdray.x), + -dimDdcoeff * (dcoeffDdcenter.y + dcoeffDdray.y), + -dimDdcoeff * (dcoeffDdcenter.z + dcoeffDdray.z)); + grad_cam->cam_pos += -dimDdcoeff * (dcoeffDdcenter + dcoeffDdray); + grad_cam->pixel_0_0_center += dimDdcoeff * dcoeffDdray; + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dimDdpixel00centerglob: %.9f, %.9f, %.9f.\n", + idx, + dimDdcoeff * dcoeffDdray.x, + dimDdcoeff * dcoeffDdray.y, + dimDdcoeff * dcoeffDdray.z); + grad_cam->pixel_dir_x += + (dimDdcoeff * static_cast(coord_x)) * dcoeffDdray; + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dimDdpixel_dir_x: %.9f, %.9f, %.9f.\n", + idx, + (dimDdcoeff * static_cast(coord_x)) * dcoeffDdray.x, + (dimDdcoeff * static_cast(coord_x)) * dcoeffDdray.y, + (dimDdcoeff * static_cast(coord_x)) * dcoeffDdray.z); + grad_cam->pixel_dir_y += + (dimDdcoeff * static_cast(coord_y)) * dcoeffDdray; + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_GRAD, + "grad %u|dimDdpixel_dir_y: %.9f, %.9f, %.9f.\n", + idx, + (dimDdcoeff * static_cast(coord_y)) * dcoeffDdray.x, + (dimDdcoeff * static_cast(coord_y)) * dcoeffDdray.y, + (dimDdcoeff * static_cast(coord_y)) * dcoeffDdray.z); + } + } + } + } + } + } + return true; +}; + +} // namespace Renderer +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.fill_bg.device.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.fill_bg.device.h new file mode 100644 index 0000000000000000000000000000000000000000..2a737d3eb609781f08120eb734982987866637f4 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.fill_bg.device.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_INCLUDE_RENDERER_FILL_BG_DEVICE_H_ +#define PULSAR_NATIVE_INCLUDE_RENDERER_FILL_BG_DEVICE_H_ + +#include "../global.h" +#include "./camera.h" +#include "./commands.h" +#include "./renderer.h" + +namespace pulsar { +namespace Renderer { + +template +GLOBAL void fill_bg( + Renderer renderer, + const CamInfo cam, + float const* const bg_col_d, + const float gamma, + const uint mode) { + GET_PARALLEL_IDS_2D(coord_x, coord_y, cam.film_width, cam.film_height); + int write_loc = coord_y * cam.film_width * (3 + 2 * renderer.n_track) + + coord_x * (3 + 2 * renderer.n_track); + if (renderer.forw_info_d[write_loc + 1] // sm_d + == 0.f) { + // This location has not been processed yet. + // Write first the forw_info: + // sm_m + renderer.forw_info_d[write_loc] = + cam.background_normalization_depth / gamma; + // sm_d + renderer.forw_info_d[write_loc + 1] = 1.f; + // max_closest_possible_intersection_hit + renderer.forw_info_d[write_loc + 2] = -1.f; + // sphere IDs and intersection depths. + for (int i = 0; i < renderer.n_track; ++i) { + int sphere_id = -1; + IASF(sphere_id, renderer.forw_info_d[write_loc + 3 + i * 2]); + renderer.forw_info_d[write_loc + 3 + i * 2 + 1] = -1.f; + } + if (mode == 0) { + // Image background. + for (int i = 0; i < cam.n_channels; ++i) { + renderer.result_d + [coord_y * cam.film_width * cam.n_channels + + coord_x * cam.n_channels + i] = bg_col_d[i]; + } + } + } + END_PARALLEL_2D_NORET(); +}; + +} // namespace Renderer +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.fill_bg.instantiate.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.fill_bg.instantiate.h new file mode 100644 index 0000000000000000000000000000000000000000..02830204a6874b8223bde1615fa9ef8ffa4d318c --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.fill_bg.instantiate.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "./renderer.fill_bg.device.h" + +namespace pulsar { +namespace Renderer { + +template GLOBAL void fill_bg( + Renderer renderer, + const CamInfo norm, + float const* const bg_col_d, + const float gamma, + const uint mode); + +} // namespace Renderer +} // namespace pulsar diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.forward.device.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.forward.device.h new file mode 100644 index 0000000000000000000000000000000000000000..3f0412f576de4dd77b3f2be6a27ff8ddb144ca74 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.forward.device.h @@ -0,0 +1,302 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_INCLUDE_RENDERER_FORWARD_DEVICE_H_ +#define PULSAR_NATIVE_INCLUDE_RENDERER_FORWARD_DEVICE_H_ + +#include "../global.h" +#include "./camera.device.h" +#include "./commands.h" +#include "./math.h" +#include "./renderer.h" + +namespace pulsar { +namespace Renderer { + +template +void forward( + Renderer* self, + const float* vert_pos, + const float* vert_col, + const float* vert_rad, + const CamInfo& cam, + const float& gamma, + float percent_allowed_difference, + const uint& max_n_hits, + const float* bg_col_d, + const float* opacity_d, + const size_t& num_balls, + const uint& mode, + cudaStream_t stream) { + ARGCHECK(gamma > 0.f && gamma <= 1.f, 6, "gamma must be in [0., 1.]"); + ARGCHECK( + percent_allowed_difference >= 0.f && percent_allowed_difference <= 1.f, + 7, + "percent_allowed_difference must be in [0., 1.]"); + ARGCHECK(max_n_hits >= 1u, 8, "max_n_hits must be >= 1"); + ARGCHECK( + num_balls > 0 && num_balls <= self->max_num_balls, + 9, + ("num_balls must be >0 and <= max num balls! (" + + std::to_string(num_balls) + " vs. " + + std::to_string(self->max_num_balls) + ")") + .c_str()); + ARGCHECK( + cam.film_width == self->cam.film_width && + cam.film_height == self->cam.film_height, + 5, + "cam result width and height must agree"); + ARGCHECK(mode <= 1, 10, "mode must be <= 1!"); + if (percent_allowed_difference > 1.f - FEPS) { + LOG(WARNING) << "percent_allowed_difference > " << (1.f - FEPS) + << "! Clamping to " << (1.f - FEPS) << "."; + percent_allowed_difference = 1.f - FEPS; + } + LOG_IF(INFO, PULSAR_LOG_RENDER) << "Rendering forward pass..."; + // Update camera and transform into a new virtual camera system with + // centered principal point and subsection rendering. + self->cam.eye = cam.eye; + self->cam.pixel_0_0_center = cam.pixel_0_0_center - cam.eye; + self->cam.pixel_dir_x = cam.pixel_dir_x; + self->cam.pixel_dir_y = cam.pixel_dir_y; + self->cam.sensor_dir_z = cam.sensor_dir_z; + self->cam.half_pixel_size = cam.half_pixel_size; + self->cam.focal_length = cam.focal_length; + self->cam.aperture_width = cam.aperture_width; + self->cam.aperture_height = cam.aperture_height; + self->cam.min_dist = cam.min_dist; + self->cam.max_dist = cam.max_dist; + self->cam.norm_fac = cam.norm_fac; + self->cam.principal_point_offset_x = cam.principal_point_offset_x; + self->cam.principal_point_offset_y = cam.principal_point_offset_y; + self->cam.film_border_left = cam.film_border_left; + self->cam.film_border_top = cam.film_border_top; +#ifdef PULSAR_TIMINGS_ENABLED + START_TIME(calc_signature); +#endif + LAUNCH_MAX_PARALLEL_1D( + calc_signature, + num_balls, + stream, + *self, + reinterpret_cast(vert_pos), + vert_col, + vert_rad, + num_balls); + CHECKLAUNCH(); +#ifdef PULSAR_TIMINGS_ENABLED + STOP_TIME(calc_signature); + START_TIME(sort); +#endif + SORT_ASCENDING_WS( + self->min_depth_d, + self->min_depth_sorted_d, + self->ids_d, + self->ids_sorted_d, + num_balls, + self->workspace_d, + self->workspace_size, + stream); + CHECKLAUNCH(); + SORT_ASCENDING_WS( + self->min_depth_d, + self->min_depth_sorted_d, + self->ii_d, + self->ii_sorted_d, + num_balls, + self->workspace_d, + self->workspace_size, + stream); + CHECKLAUNCH(); + SORT_ASCENDING_WS( + self->min_depth_d, + self->min_depth_sorted_d, + self->di_d, + self->di_sorted_d, + num_balls, + self->workspace_d, + self->workspace_size, + stream); + CHECKLAUNCH(); +#ifdef PULSAR_TIMINGS_ENABLED + STOP_TIME(sort); + START_TIME(minmax); +#endif + IntersectInfo pixel_minmax; + pixel_minmax.min.x = MAX_USHORT; + pixel_minmax.min.y = MAX_USHORT; + pixel_minmax.max.x = 0; + pixel_minmax.max.y = 0; + REDUCE_WS( + self->ii_sorted_d, + self->min_max_pixels_d, + num_balls, + IntersectInfoMinMax(), + pixel_minmax, + self->workspace_d, + self->workspace_size, + stream); + COPY_DEV_HOST(&pixel_minmax, self->min_max_pixels_d, IntersectInfo, 1); + LOG_IF(INFO, PULSAR_LOG_RENDER) + << "Region with pixels to render: " << pixel_minmax.min.x << ":" + << pixel_minmax.max.x << " (x), " << pixel_minmax.min.y << ":" + << pixel_minmax.max.y << " (y)."; +#ifdef PULSAR_TIMINGS_ENABLED + STOP_TIME(minmax); + START_TIME(render); +#endif + MEMSET( + self->result_d, + 0, + float, + self->cam.film_width * self->cam.film_height * self->cam.n_channels, + stream); + MEMSET( + self->forw_info_d, + 0, + float, + self->cam.film_width * self->cam.film_height * (3 + 2 * self->n_track), + stream); + if (pixel_minmax.max.y > pixel_minmax.min.y && + pixel_minmax.max.x > pixel_minmax.min.x) { + PASSERT( + pixel_minmax.min.x >= static_cast(self->cam.film_border_left) && + pixel_minmax.min.x < + static_cast( + self->cam.film_border_left + self->cam.film_width) && + pixel_minmax.max.x <= + static_cast( + self->cam.film_border_left + self->cam.film_width) && + pixel_minmax.min.y >= static_cast(self->cam.film_border_top) && + pixel_minmax.min.y < + static_cast( + self->cam.film_border_top + self->cam.film_height) && + pixel_minmax.max.y <= + static_cast( + self->cam.film_border_top + self->cam.film_height)); + // Cut the image in 3x3 regions. + int y_step = RENDER_BLOCK_SIZE * + iDivCeil(pixel_minmax.max.y - pixel_minmax.min.y, + 3u * RENDER_BLOCK_SIZE); + int x_step = RENDER_BLOCK_SIZE * + iDivCeil(pixel_minmax.max.x - pixel_minmax.min.x, + 3u * RENDER_BLOCK_SIZE); + LOG_IF(INFO, PULSAR_LOG_RENDER) << "Using image slices of size " << x_step + << ", " << y_step << " (W, H)."; + for (int y_min = pixel_minmax.min.y; y_min < pixel_minmax.max.y; + y_min += y_step) { + for (int x_min = pixel_minmax.min.x; x_min < pixel_minmax.max.x; + x_min += x_step) { + // Create region selection. + LAUNCH_MAX_PARALLEL_1D( + create_selector, + num_balls, + stream, + self->ii_sorted_d, + num_balls, + x_min, + x_min + x_step, + y_min, + y_min + y_step, + self->region_flags_d); + CHECKLAUNCH(); + SELECT_FLAGS_WS( + self->region_flags_d, + self->ii_sorted_d, + self->ii_d, + self->num_selected_d, + num_balls, + self->workspace_d, + self->workspace_size, + stream); + CHECKLAUNCH(); + SELECT_FLAGS_WS( + self->region_flags_d, + self->di_sorted_d, + self->di_d, + self->num_selected_d, + num_balls, + self->workspace_d, + self->workspace_size, + stream); + CHECKLAUNCH(); + SELECT_FLAGS_WS( + self->region_flags_d, + self->ids_sorted_d, + self->ids_d, + self->num_selected_d, + num_balls, + self->workspace_d, + self->workspace_size, + stream); + CHECKLAUNCH(); + LAUNCH_PARALLEL_2D( + render, + x_step, + y_step, + RENDER_BLOCK_SIZE, + RENDER_BLOCK_SIZE, + stream, + self->num_selected_d, + self->ii_d, + self->di_d, + self->min_depth_d, + self->ids_d, + opacity_d, + self->cam, + gamma, + percent_allowed_difference, + max_n_hits, + bg_col_d, + mode, + x_min, + y_min, + x_step, + y_step, + self->result_d, + self->forw_info_d, + self->n_track); + CHECKLAUNCH(); + } + } + } + if (mode == 0) { + LAUNCH_MAX_PARALLEL_2D( + fill_bg, + static_cast(self->cam.film_width), + static_cast(self->cam.film_height), + stream, + *self, + self->cam, + bg_col_d, + gamma, + mode); + CHECKLAUNCH(); + } +#ifdef PULSAR_TIMINGS_ENABLED + STOP_TIME(render); + float time_ms; + // This blocks the result and prevents batch-processing from parallelizing. + GET_TIME(calc_signature, &time_ms); + std::cout << "Time for signature calculation: " << time_ms << " ms" + << std::endl; + GET_TIME(sort, &time_ms); + std::cout << "Time for sorting: " << time_ms << " ms" << std::endl; + GET_TIME(minmax, &time_ms); + std::cout << "Time for minmax pixel calculation: " << time_ms << " ms" + << std::endl; + GET_TIME(render, &time_ms); + std::cout << "Time for rendering: " << time_ms << " ms" << std::endl; +#endif + LOG_IF(INFO, PULSAR_LOG_RENDER) << "Forward pass complete."; +} + +} // namespace Renderer +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.forward.instantiate.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.forward.instantiate.h new file mode 100644 index 0000000000000000000000000000000000000000..7f57bc8681b7c7f1356f3c3e134595ab2d1955f0 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.forward.instantiate.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "./renderer.forward.device.h" + +namespace pulsar { +namespace Renderer { + +template void forward( + Renderer* self, + const float* vert_pos, + const float* vert_col, + const float* vert_rad, + const CamInfo& cam, + const float& gamma, + float percent_allowed_difference, + const uint& max_n_hits, + const float* bg_col_d, + const float* opacity_d, + const size_t& num_balls, + const uint& mode, + cudaStream_t stream); + +} // namespace Renderer +} // namespace pulsar diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.get_screen_area.device.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.get_screen_area.device.h new file mode 100644 index 0000000000000000000000000000000000000000..1a85a1bd20cfa0773e395163871ea5a7a8b39347 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.get_screen_area.device.h @@ -0,0 +1,144 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_INCLUDE_RENDERER_GET_SCREEN_AREA_DEVICE_H_ +#define PULSAR_NATIVE_INCLUDE_RENDERER_GET_SCREEN_AREA_DEVICE_H_ + +#include "../global.h" +#include "./camera.device.h" +#include "./commands.h" +#include "./math.h" + +namespace pulsar { +namespace Renderer { + +/** + * Find the closest enclosing screen area rectangle in pixels that encloses a + * ball. + * + * The method returns the two x and the two y values of the boundaries. They + * are not ordered yet and you need to find min and max for the left/right and + * lower/upper boundary. + * + * The return values are floats and need to be rounded appropriately. + */ +INLINE DEVICE bool get_screen_area( + const float3& ball_center_cam, + const float3& ray_center_norm, + const float& vert_rad, + const CamInfo& cam, + const uint& idx, + /* Out variables. */ + float* x_1, + float* x_2, + float* y_1, + float* y_2) { + float cos_alpha = dot(cam.sensor_dir_z, ray_center_norm); + float2 o__c_, alpha, theta; + if (cos_alpha < EPS) { + PULSAR_LOG_DEV( + PULSAR_LOG_CALC_SIGNATURE, + "signature %d|ball not visible. cos_alpha: %.9f.\n", + idx, + cos_alpha); + // No intersection, ball won't be visible. + return false; + } + // Multiply the direction vector with the camera rotation matrix + // to have the optical axis being the canonical z vector (0, 0, 1). + // TODO: optimize. + const float3 ball_center_cam_rot = rotate( + ball_center_cam, + cam.pixel_dir_x / length(cam.pixel_dir_x), + cam.pixel_dir_y / length(cam.pixel_dir_y), + cam.sensor_dir_z); + PULSAR_LOG_DEV( + PULSAR_LOG_CALC_SIGNATURE, + "signature %d|ball_center_cam_rot: %f, %f, %f.\n", + idx, + ball_center_cam.x, + ball_center_cam.y, + ball_center_cam.z); + const float pixel_size_norm_fac = FRCP(2.f * cam.half_pixel_size); + const float optical_offset_x = + (static_cast(cam.aperture_width) - 1.f) * .5f; + const float optical_offset_y = + (static_cast(cam.aperture_height) - 1.f) * .5f; + if (cam.orthogonal_projection) { + *x_1 = + FMA(ball_center_cam_rot.x - vert_rad, + pixel_size_norm_fac, + optical_offset_x); + *x_2 = + FMA(ball_center_cam_rot.x + vert_rad, + pixel_size_norm_fac, + optical_offset_x); + *y_1 = + FMA(ball_center_cam_rot.y - vert_rad, + pixel_size_norm_fac, + optical_offset_y); + *y_2 = + FMA(ball_center_cam_rot.y + vert_rad, + pixel_size_norm_fac, + optical_offset_y); + return true; + } else { + o__c_.x = FMAX( + FSQRT( + ball_center_cam_rot.x * ball_center_cam_rot.x + + ball_center_cam_rot.z * ball_center_cam_rot.z), + FEPS); + o__c_.y = FMAX( + FSQRT( + ball_center_cam_rot.y * ball_center_cam_rot.y + + ball_center_cam_rot.z * ball_center_cam_rot.z), + FEPS); + PULSAR_LOG_DEV( + PULSAR_LOG_CALC_SIGNATURE, + "signature %d|o__c_: %f, %f.\n", + idx, + o__c_.x, + o__c_.y); + alpha.x = sign_dir(ball_center_cam_rot.x) * + acos(FMIN(FMAX(ball_center_cam_rot.z / o__c_.x, -1.f), 1.f)); + alpha.y = -sign_dir(ball_center_cam_rot.y) * + acos(FMIN(FMAX(ball_center_cam_rot.z / o__c_.y, -1.f), 1.f)); + theta.x = asin(FMIN(FMAX(vert_rad / o__c_.x, -1.f), 1.f)); + theta.y = asin(FMIN(FMAX(vert_rad / o__c_.y, -1.f), 1.f)); + PULSAR_LOG_DEV( + PULSAR_LOG_CALC_SIGNATURE, + "signature %d|alpha.x: %f, alpha.y: %f, theta.x: %f, theta.y: %f.\n", + idx, + alpha.x, + alpha.y, + theta.x, + theta.y); + *x_1 = tan(alpha.x - theta.x) * cam.focal_length; + *x_2 = tan(alpha.x + theta.x) * cam.focal_length; + *y_1 = tan(alpha.y - theta.y) * cam.focal_length; + *y_2 = tan(alpha.y + theta.y) * cam.focal_length; + PULSAR_LOG_DEV( + PULSAR_LOG_CALC_SIGNATURE, + "signature %d|in sensor plane: x_1: %f, x_2: %f, y_1: %f, y_2: %f.\n", + idx, + *x_1, + *x_2, + *y_1, + *y_2); + *x_1 = FMA(*x_1, pixel_size_norm_fac, optical_offset_x); + *x_2 = FMA(*x_2, pixel_size_norm_fac, optical_offset_x); + *y_1 = FMA(*y_1, -pixel_size_norm_fac, optical_offset_y); + *y_2 = FMA(*y_2, -pixel_size_norm_fac, optical_offset_y); + return true; + } +}; + +} // namespace Renderer +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.h new file mode 100644 index 0000000000000000000000000000000000000000..d6755ee91887b8f6316563b03cee9c524a6f7315 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.h @@ -0,0 +1,468 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_INCLUDE_RENDERER_H_ +#define PULSAR_NATIVE_INCLUDE_RENDERER_H_ + +#include + +#include "../global.h" +#include "./camera.h" + +namespace pulsar { +namespace Renderer { + +//! Remember to order struct members from larger size to smaller size +//! to avoid padding (for more info, see for example here: +//! http://www.catb.org/esr/structure-packing/). + +/** + * This is the information that's needed to do a fast screen point + * intersection with one of the balls. + * + * Aim to keep this below 8 bytes (256 bytes per cache-line / 32 threads in a + * warp = 8 bytes per thread). + */ +struct IntersectInfo { + ushort2 min; /** minimum x, y in pixel coordinates. */ + ushort2 max; /** maximum x, y in pixel coordinates. */ +}; +static_assert( + sizeof(IntersectInfo) == 8, + "The compiled size of `IntersectInfo` is wrong."); + +/** + * Reduction operation to find the limits of multiple IntersectInfo objects. + */ +struct IntersectInfoMinMax { + IHD IntersectInfo + operator()(const IntersectInfo& a, const IntersectInfo& b) const { + // Treat the special case of an invalid intersect info object or one for + // a ball out of bounds. + if (b.max.x == MAX_USHORT && b.min.x == MAX_USHORT && + b.max.y == MAX_USHORT && b.min.y == MAX_USHORT) { + return a; + } + if (a.max.x == MAX_USHORT && a.min.x == MAX_USHORT && + a.max.y == MAX_USHORT && a.min.y == MAX_USHORT) { + return b; + } + IntersectInfo result; + result.min.x = std::min(a.min.x, b.min.x); + result.min.y = std::min(a.min.y, b.min.y); + result.max.x = std::max(a.max.x, b.max.x); + result.max.y = std::max(a.max.y, b.max.y); + return result; + } +}; + +/** + * All information that's needed to draw a ball. + * + * It's necessary to keep this information in float (not half) format, + * because the loss in accuracy would be too high and lead to artifacts. + */ +struct DrawInfo { + float3 ray_center_norm; /** Ray to the ball center, normalized. */ + /** Ball color. + * + * This might be the full color in the case of n_channels <= 3. Otherwise, + * a pointer to the original 'color' data is stored in the following union. + */ + float first_color; + union { + float color[2]; + float* ptr; + } color_union; + float t_center; /** Distance from the camera to the ball center. */ + float radius; /** Ball radius. */ +}; +static_assert( + sizeof(DrawInfo) == 8 * 4, + "The compiled size of `DrawInfo` is wrong."); + +/** + * An object to collect all associated data with the renderer. + * + * The `_d` suffixed pointers point to memory 'on-device', potentially on the + * GPU. All other variables are expected to point to CPU memory. + */ +struct Renderer { + /** Dummy initializer to make sure all pointers are set to NULL to + * be safe for the device-specific 'construct' and 'destruct' methods. + */ + inline Renderer() { + max_num_balls = 0; + result_d = NULL; + min_depth_d = NULL; + min_depth_sorted_d = NULL; + ii_d = NULL; + ii_sorted_d = NULL; + ids_d = NULL; + ids_sorted_d = NULL; + workspace_d = NULL; + di_d = NULL; + di_sorted_d = NULL; + region_flags_d = NULL; + num_selected_d = NULL; + forw_info_d = NULL; + grad_pos_d = NULL; + grad_col_d = NULL; + grad_rad_d = NULL; + grad_cam_d = NULL; + grad_opy_d = NULL; + grad_cam_buf_d = NULL; + n_grad_contributions_d = NULL; + }; + /** The camera for this renderer. In world-coordinates. */ + CamInfo cam; + /** + * The maximum amount of balls the renderer can handle. Resources are + * pre-allocated to account for this size. Less than this amount of balls + * can be rendered, but not more. + */ + int max_num_balls; + /** The result buffer. */ + float* result_d; + /** Closest possible intersection depth per sphere w.r.t. the camera. */ + float* min_depth_d; + /** Closest possible intersection depth per sphere, ordered ascending. */ + float* min_depth_sorted_d; + /** The intersect infos per sphere. */ + IntersectInfo* ii_d; + /** The intersect infos per sphere, ordered by their closest possible + * intersection depth (asc.). */ + IntersectInfo* ii_sorted_d; + /** Original sphere IDs. */ + int* ids_d; + /** Original sphere IDs, ordered by their closest possible intersection depth + * (asc.). */ + int* ids_sorted_d; + /** Workspace for CUB routines. */ + char* workspace_d; + /** Workspace size for CUB routines. */ + size_t workspace_size; + /** The draw information structures for each sphere. */ + DrawInfo* di_d; + /** The draw information structures sorted by closest possible intersection + * depth (asc.). */ + DrawInfo* di_sorted_d; + /** Region association buffer. */ + char* region_flags_d; + /** Num spheres in the current region. */ + size_t* num_selected_d; + /** Pointer to information from the forward pass. */ + float* forw_info_d; + /** Struct containing information about the min max pixels that contain + * rendered information in the image. */ + IntersectInfo* min_max_pixels_d; + /** Gradients w.r.t. position. */ + float3* grad_pos_d; + /** Gradients w.r.t. color. */ + float* grad_col_d; + /** Gradients w.r.t. radius. */ + float* grad_rad_d; + /** Gradients w.r.t. camera parameters. */ + float* grad_cam_d; + /** Gradients w.r.t. opacity. */ + float* grad_opy_d; + /** Camera gradient information by sphere. + * + * Here, every sphere's contribution to the camera gradients is stored. It is + * aggregated and written to grad_cam_d in a separate step. This avoids write + * conflicts when processing the spheres. + */ + CamGradInfo* grad_cam_buf_d; + /** Total of all gradient contributions for this image. */ + int* n_grad_contributions_d; + /** The number of spheres to track for backpropagation. */ + int n_track; +}; + +inline bool operator==(const Renderer& a, const Renderer& b) { + return a.cam == b.cam && a.max_num_balls == b.max_num_balls; +} + +/** + * Construct a renderer. + */ +template +void construct( + Renderer* self, + const size_t& max_num_balls, + const int& width, + const int& height, + const bool& orthogonal_projection, + const bool& right_handed_system, + const float& background_normalization_depth, + const uint& n_channels, + const uint& n_track); + +/** + * Destruct the renderer and free the associated memory. + */ +template +void destruct(Renderer* self); + +/** + * Create a selection of points inside a rectangle. + * + * This write boolen values into `region_flags_d', which can + * for example be used by a CUB function to extract the selection. + */ +template +GLOBAL void create_selector( + IntersectInfo const* const RESTRICT ii_sorted_d, + const uint num_balls, + const int min_x, + const int max_x, + const int min_y, + const int max_y, + /* Out variables. */ + char* RESTRICT region_flags_d); + +/** + * Calculate a signature for a ball. + * + * Populate the `ids_d`, `ii_d`, `di_d` and `min_depth_d` fields of the + * renderer. For spheres not visible in the image, sets the id field to -1, + * min_depth_d to MAX_FLOAT and the ii_d.min.x fields to MAX_USHORT. + */ +template +GLOBAL void calc_signature( + Renderer renderer, + float3 const* const RESTRICT vert_poss, + float const* const RESTRICT vert_cols, + float const* const RESTRICT vert_rads, + const uint num_balls); + +/** + * The block size for rendering. + * + * This should be as large as possible, but is limited due to the amount + * of variables we use and the memory required per thread. + */ +#define RENDER_BLOCK_SIZE 16 +/** + * The buffer size of spheres to be loaded and analyzed for relevance. + * + * This must be at least RENDER_BLOCK_SIZE * RENDER_BLOCK_SIZE so that + * for every iteration through the loading loop every thread could add a + * 'hit' to the buffer. + */ +#define RENDER_BUFFER_SIZE RENDER_BLOCK_SIZE* RENDER_BLOCK_SIZE * 2 +/** + * The threshold after which the spheres that are in the render buffer + * are rendered and the buffer is flushed. + * + * Must be less than RENDER_BUFFER_SIZE. + */ +#define RENDER_BUFFER_LOAD_THRESH 16 * 4 + +/** + * The render function. + * + * Assumptions: + * * the focal length is appropriately chosen, + * * ray_dir_norm.z is > EPS. + * * to be completed... + */ +template +GLOBAL void render( + size_t const* const RESTRICT + num_balls, /** Number of balls relevant for this pass. */ + IntersectInfo const* const RESTRICT ii_d, /** Intersect information. */ + DrawInfo const* const RESTRICT di_d, /** Draw information. */ + float const* const RESTRICT min_depth_d, /** Minimum depth per sphere. */ + int const* const RESTRICT id_d, /** IDs. */ + float const* const RESTRICT op_d, /** Opacity. */ + const CamInfo cam_norm, /** Camera normalized with all vectors to be in the + * camera coordinate system. + */ + const float gamma, /** Transparency parameter. **/ + const float percent_allowed_difference, /** Maximum allowed + error in color. */ + const uint max_n_hits, + const float* bg_col_d, + const uint mode, + const int x_min, + const int y_min, + const int x_step, + const int y_step, + // Out variables. + float* const RESTRICT result_d, /** The result image. */ + float* const RESTRICT forw_info_d, /** Additional information needed for the + grad computation. */ + // Infrastructure. + const int n_track /** The number of spheres to track. */ +); + +/** + * Makes sure to paint background information. + * + * This is required as a separate post-processing step because certain + * pixels may not be processed during the forward pass if there is no + * possibility for a sphere to be present at their location. + */ +template +GLOBAL void fill_bg( + Renderer renderer, + const CamInfo norm, + float const* const bg_col_d, + const float gamma, + const uint mode); + +/** + * Rendering forward pass. + * + * Takes a renderer and sphere data as inputs and creates a rendering. + */ +template +void forward( + Renderer* self, + const float* vert_pos, + const float* vert_col, + const float* vert_rad, + const CamInfo& cam, + const float& gamma, + float percent_allowed_difference, + const uint& max_n_hits, + const float* bg_col_d, + const float* opacity_d, + const size_t& num_balls, + const uint& mode, + cudaStream_t stream); + +/** + * Normalize the camera gradients by the number of spheres that contributed. + */ +template +GLOBAL void norm_cam_gradients(Renderer renderer); + +/** + * Normalize the sphere gradients. + * + * We're assuming that the samples originate from a Monte Carlo + * sampling process and normalize by number and sphere area. + */ +template +GLOBAL void norm_sphere_gradients(Renderer renderer, const int num_balls); + +#define GRAD_BLOCK_SIZE 16 +/** Calculate the gradients. + */ +template +GLOBAL void calc_gradients( + const CamInfo cam, /** Camera in world coordinates. */ + float const* const RESTRICT grad_im, /** The gradient image. */ + const float + gamma, /** The transparency parameter used in the forward pass. */ + float3 const* const RESTRICT vert_poss, /** Vertex position vector. */ + float const* const RESTRICT vert_cols, /** Vertex color vector. */ + float const* const RESTRICT vert_rads, /** Vertex radius vector. */ + float const* const RESTRICT opacity, /** Vertex opacity. */ + const uint num_balls, /** Number of balls. */ + float const* const RESTRICT result_d, /** Result image. */ + float const* const RESTRICT forw_info_d, /** Forward pass info. */ + DrawInfo const* const RESTRICT di_d, /** Draw information. */ + IntersectInfo const* const RESTRICT ii_d, /** Intersect information. */ + // Mode switches. + const bool calc_grad_pos, + const bool calc_grad_col, + const bool calc_grad_rad, + const bool calc_grad_cam, + const bool calc_grad_opy, + // Out variables. + float* const RESTRICT grad_rad_d, /** Radius gradients. */ + float* const RESTRICT grad_col_d, /** Color gradients. */ + float3* const RESTRICT grad_pos_d, /** Position gradients. */ + CamGradInfo* const RESTRICT grad_cam_buf_d, /** Camera gradient buffer. */ + float* const RESTRICT grad_opy_d, /** Opacity gradient buffer. */ + int* const RESTRICT + grad_contributed_d, /** Gradient contribution counter. */ + // Infrastructure. + const int n_track, + const uint offs_x = 0, + const uint offs_y = 0); + +/** + * A full backward pass. + * + * Creates the gradients for the given gradient_image and the spheres. + */ +template +void backward( + Renderer* self, + const float* grad_im, + const float* image, + const float* forw_info, + const float* vert_pos, + const float* vert_col, + const float* vert_rad, + const CamInfo& cam, + const float& gamma, + float percent_allowed_difference, + const uint& max_n_hits, + const float* vert_opy, + const size_t& num_balls, + const uint& mode, + const bool& dif_pos, + const bool& dif_col, + const bool& dif_rad, + const bool& dif_cam, + const bool& dif_opy, + cudaStream_t stream); + +/** + * A debug backward pass. + * + * This is a function to debug the gradient calculation. It calculates the + * gradients for exactly one pixel (set with pos_x and pos_y) without averaging. + * + * *Uses only the first sphere for camera gradient calculation!* + */ +template +void backward_dbg( + Renderer* self, + const float* grad_im, + const float* image, + const float* forw_info, + const float* vert_pos, + const float* vert_col, + const float* vert_rad, + const CamInfo& cam, + const float& gamma, + float percent_allowed_difference, + const uint& max_n_hits, + const float* vert_opy, + const size_t& num_balls, + const uint& mode, + const bool& dif_pos, + const bool& dif_col, + const bool& dif_rad, + const bool& dif_cam, + const bool& dif_opy, + const uint& pos_x, + const uint& pos_y, + cudaStream_t stream); + +template +void nn( + const float* ref_ptr, + const float* tar_ptr, + const uint& k, + const uint& d, + const uint& n, + float* dist_ptr, + int32_t* inds_ptr, + cudaStream_t stream); + +} // namespace Renderer +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.norm_cam_gradients.device.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.norm_cam_gradients.device.h new file mode 100644 index 0000000000000000000000000000000000000000..e1dfd55d0b1363c1d8d38709460e00a75efeef5a --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.norm_cam_gradients.device.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_INCLUDE_RENDERER_NORM_CAM_GRADIENTS_DEVICE_H_ +#define PULSAR_NATIVE_INCLUDE_RENDERER_NORM_CAM_GRADIENTS_DEVICE_H_ + +#include "../global.h" +#include "./camera.device.h" +#include "./commands.h" +#include "./math.h" +#include "./renderer.h" + +namespace pulsar { +namespace Renderer { + +/** + * Normalize the camera gradients by the number of spheres that contributed. + */ +template +GLOBAL void norm_cam_gradients(Renderer renderer) { + GET_PARALLEL_IDX_1D(idx, 1); + CamGradInfo* cgi = reinterpret_cast(renderer.grad_cam_d); + *cgi = *cgi * FRCP(static_cast(*renderer.n_grad_contributions_d)); + END_PARALLEL_NORET(); +}; + +} // namespace Renderer +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.norm_cam_gradients.instantiate.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.norm_cam_gradients.instantiate.h new file mode 100644 index 0000000000000000000000000000000000000000..98e05a67e470237a9328d7a441e7b700a7ce675d --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.norm_cam_gradients.instantiate.h @@ -0,0 +1,17 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "./renderer.norm_cam_gradients.device.h" + +namespace pulsar { +namespace Renderer { + +template GLOBAL void norm_cam_gradients(Renderer renderer); + +} // namespace Renderer +} // namespace pulsar diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.norm_sphere_gradients.device.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.norm_sphere_gradients.device.h new file mode 100644 index 0000000000000000000000000000000000000000..37e0eb00a5179911216a5d2827feb83ade487755 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.norm_sphere_gradients.device.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_INCLUDE_RENDERER_NORM_SPHERE_GRADIENTS_H_ +#define PULSAR_NATIVE_INCLUDE_RENDERER_NORM_SPHERE_GRADIENTS_H_ + +#include "../global.h" +#include "./commands.h" +#include "./math.h" +#include "./renderer.h" + +namespace pulsar { +namespace Renderer { + +/** + * Normalize the sphere gradients. + * + * We're assuming that the samples originate from a Monte Carlo + * sampling process and normalize by number and sphere area. + */ +template +GLOBAL void norm_sphere_gradients(Renderer renderer, const int num_balls) { + GET_PARALLEL_IDX_1D(idx, num_balls); + float norm_fac = 0.f; + IntersectInfo ii; + if (renderer.ids_sorted_d[idx] > 0) { + ii = renderer.ii_d[idx]; + // Normalize the sphere gradients as averages. + // This avoids the case that there are small spheres in a scene with still + // un-converged colors whereas the big spheres already converged, just + // because their integrated learning rate is 'higher'. + norm_fac = FRCP(static_cast(renderer.ids_sorted_d[idx])); + } + PULSAR_LOG_DEV_NODE( + PULSAR_LOG_NORMALIZE, + "ids_sorted_d[idx]: %d, norm_fac: %.9f.\n", + renderer.ids_sorted_d[idx], + norm_fac); + renderer.grad_rad_d[idx] *= norm_fac; + for (uint c_idx = 0; c_idx < renderer.cam.n_channels; ++c_idx) { + renderer.grad_col_d[idx * renderer.cam.n_channels + c_idx] *= norm_fac; + } + renderer.grad_pos_d[idx] *= norm_fac; + renderer.grad_opy_d[idx] *= norm_fac; + + if (renderer.ids_sorted_d[idx] > 0) { + // For the camera, we need to be more correct and have the gradients + // be proportional to the area they cover in the image. + // This leads to a formulation very much like in monte carlo integration: + norm_fac = FRCP(static_cast(renderer.ids_sorted_d[idx])) * + (static_cast(ii.max.x) - static_cast(ii.min.x)) * + (static_cast(ii.max.y) - static_cast(ii.min.y)) * + 1e-3f; // for better numerics. + } + renderer.grad_cam_buf_d[idx].cam_pos *= norm_fac; + renderer.grad_cam_buf_d[idx].pixel_0_0_center *= norm_fac; + renderer.grad_cam_buf_d[idx].pixel_dir_x *= norm_fac; + renderer.grad_cam_buf_d[idx].pixel_dir_y *= norm_fac; + // The sphere only contributes to the camera gradients if it is + // large enough in screen space. + if (renderer.ids_sorted_d[idx] > 0 && ii.max.x >= ii.min.x + 3 && + ii.max.y >= ii.min.y + 3) + renderer.ids_sorted_d[idx] = 1; + END_PARALLEL_NORET(); +}; + +} // namespace Renderer +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.norm_sphere_gradients.instantiate.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.norm_sphere_gradients.instantiate.h new file mode 100644 index 0000000000000000000000000000000000000000..bedcf81611cb20b2b404776f477cb3fe174608d2 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.norm_sphere_gradients.instantiate.h @@ -0,0 +1,19 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "./renderer.norm_sphere_gradients.device.h" + +namespace pulsar { +namespace Renderer { + +template GLOBAL void norm_sphere_gradients( + Renderer renderer, + const int num_balls); + +} // namespace Renderer +} // namespace pulsar diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.render.device.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.render.device.h new file mode 100644 index 0000000000000000000000000000000000000000..d1fe23f4ab5ac2723607c711268ece6954715480 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.render.device.h @@ -0,0 +1,416 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_INCLUDE_RENDERER_RENDER_DEVICE_H_ +#define PULSAR_NATIVE_INCLUDE_RENDERER_RENDER_DEVICE_H_ + +#include "../global.h" +#include "./camera.device.h" +#include "./commands.h" +#include "./math.h" +#include "./renderer.h" + +#include "./closest_sphere_tracker.device.h" +#include "./renderer.draw.device.h" + +namespace pulsar { +namespace Renderer { + +template +GLOBAL void render( + size_t const* const RESTRICT + num_balls, /** Number of balls relevant for this pass. */ + IntersectInfo const* const RESTRICT ii_d, /** Intersect information. */ + DrawInfo const* const RESTRICT di_d, /** Draw information. */ + float const* const RESTRICT min_depth_d, /** Minimum depth per sphere. */ + int const* const RESTRICT ids_d, /** IDs. */ + float const* const RESTRICT op_d, /** Opacity. */ + const CamInfo cam_norm, /** Camera normalized with all vectors to be in the + * camera coordinate system. + */ + const float gamma, /** Transparency parameter. **/ + const float percent_allowed_difference, /** Maximum allowed + error in color. */ + const uint max_n_hits, + const float* bg_col, + const uint mode, + const int x_min, + const int y_min, + const int x_step, + const int y_step, + // Out variables. + float* const RESTRICT result_d, /** The result image. */ + float* const RESTRICT forw_info_d, /** Additional information needed for the + grad computation. */ + const int n_track /** The number of spheres to track for backprop. */ +) { + // Do not early stop threads in this block here. They can all contribute to + // the scanning process, we just have to prevent from writing their result. + GET_PARALLEL_IDS_2D(offs_x, offs_y, x_step, y_step); + // Variable declarations and const initializations. + const float ln_pad_over_1minuspad = + FLN(percent_allowed_difference / (1.f - percent_allowed_difference)); + /** A facility to track the closest spheres to the camera + (in preparation for gradient calculation). */ + ClosestSphereTracker tracker(n_track); + const uint coord_x = x_min + offs_x; /** Ray coordinate x. */ + const uint coord_y = y_min + offs_y; /** Ray coordinate y. */ + float3 ray_dir_norm; /** Ray cast through the pixel, normalized. */ + float2 projected_ray; /** Ray intersection with the sensor. */ + if (cam_norm.orthogonal_projection) { + ray_dir_norm = cam_norm.sensor_dir_z; + projected_ray.x = static_cast(coord_x); + projected_ray.y = static_cast(coord_y); + } else { + ray_dir_norm = normalize( + cam_norm.pixel_0_0_center + coord_x * cam_norm.pixel_dir_x + + coord_y * cam_norm.pixel_dir_y); + // This is a reasonable assumption for normal focal lengths and image sizes. + PASSERT(FABS(ray_dir_norm.z) > FEPS); + projected_ray.x = ray_dir_norm.x / ray_dir_norm.z * cam_norm.focal_length; + projected_ray.y = ray_dir_norm.y / ray_dir_norm.z * cam_norm.focal_length; + } + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_RENDER_PIX, + "render|ray_dir_norm: %.9f, %.9f, %.9f. projected_ray: %.9f, %.9f.\n", + ray_dir_norm.x, + ray_dir_norm.y, + ray_dir_norm.z, + projected_ray.x, + projected_ray.y); + // Set up shared infrastructure. + /** This entire thread block. */ + cg::thread_block thread_block = cg::this_thread_block(); + /** The collaborators within a warp. */ + cg::coalesced_group thread_warp = cg::coalesced_threads(); + /** The number of loaded balls in the load buffer di_l. */ + SHARED uint n_loaded; + /** Draw information buffer. */ + SHARED DrawInfo di_l[RENDER_BUFFER_SIZE]; + /** The original sphere id of each loaded sphere. */ + SHARED uint sphere_id_l[RENDER_BUFFER_SIZE]; + /** The number of pixels in this block that are done. */ + SHARED int n_pixels_done; + /** Whether loading of balls is completed. */ + SHARED bool loading_done; + /** The number of balls loaded overall (just for statistics). */ + [[maybe_unused]] SHARED int n_balls_loaded; + /** The area this thread block covers. */ + SHARED IntersectInfo block_area; + if (thread_block.thread_rank() == 0) { + // Initialize the shared variables. + n_loaded = 0; + block_area.min.x = static_cast(coord_x); + block_area.max.x = static_cast(IMIN( + coord_x + blockDim.x, cam_norm.film_border_left + cam_norm.film_width)); + block_area.min.y = static_cast(coord_y); + block_area.max.y = static_cast(IMIN( + coord_y + blockDim.y, cam_norm.film_border_top + cam_norm.film_height)); + n_pixels_done = 0; + loading_done = false; + n_balls_loaded = 0; + } + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_RENDER_PIX, + "render|block_area.min: %d, %d. block_area.max: %d, %d.\n", + block_area.min.x, + block_area.min.y, + block_area.max.x, + block_area.max.y); + // Initialization of the pixel with the background color. + /** + * The result of this very pixel. + * the offset calculation might overflow if this thread is out of + * bounds of the film. However, in this case result is not + * accessed, so this is fine. + */ + float* result = result_d + + (coord_y - cam_norm.film_border_top) * cam_norm.film_width * + cam_norm.n_channels + + (coord_x - cam_norm.film_border_left) * cam_norm.n_channels; + if (coord_x >= cam_norm.film_border_left && + coord_x < cam_norm.film_border_left + cam_norm.film_width && + coord_y >= cam_norm.film_border_top && + coord_y < cam_norm.film_border_top + cam_norm.film_height) { + // Initialize the result. + if (mode == 0u) { + for (uint c_id = 0; c_id < cam_norm.n_channels; ++c_id) + result[c_id] = bg_col[c_id]; + } else { + result[0] = 0.f; + } + } + /** Normalization denominator. */ + float sm_d = 1.f; + /** Normalization tracker for stable softmax. The maximum observed value. */ + float sm_m = cam_norm.background_normalization_depth / gamma; + /** Whether this pixel has had all information needed for drawing. */ + bool done = + (coord_x < cam_norm.film_border_left || + coord_x >= cam_norm.film_border_left + cam_norm.film_width || + coord_y < cam_norm.film_border_top || + coord_y >= cam_norm.film_border_top + cam_norm.film_height); + /** The depth threshold for a new point to have at least + * `percent_allowed_difference` influence on the result color. All points that + * are further away than this are ignored. + */ + float depth_threshold = done ? -1.f : MAX_FLOAT; + /** The closest intersection possible of a ball that was hit by this pixel + * ray. */ + float max_closest_possible_intersection_hit = -1.f; + bool hit; /** Whether a sphere was hit. */ + float intersection_depth; /** The intersection_depth for a sphere at this + pixel. */ + float closest_possible_intersection; /** The closest possible intersection + for this sphere. */ + float max_closest_possible_intersection; + // Sync up threads so that everyone is similarly initialized. + thread_block.sync(); + //! Coalesced loading and intersection analysis of balls. + for (uint ball_idx = thread_block.thread_rank(); + ball_idx < iDivCeil(static_cast(*num_balls), thread_block.size()) * + thread_block.size() && + !loading_done && n_pixels_done < thread_block.size(); + ball_idx += thread_block.size()) { + if (ball_idx < static_cast(*num_balls)) { // Account for overflow. + const IntersectInfo& ii = ii_d[ball_idx]; + hit = (ii.min.x <= block_area.max.x) && (ii.max.x > block_area.min.x) && + (ii.min.y <= block_area.max.y) && (ii.max.y > block_area.min.y); + if (hit) { + uint write_idx = ATOMICADD_B(&n_loaded, 1u); + di_l[write_idx] = di_d[ball_idx]; + sphere_id_l[write_idx] = static_cast(ids_d[ball_idx]); + PULSAR_LOG_DEV_PIXB( + PULSAR_LOG_RENDER_PIX, + "render|found intersection with sphere %u.\n", + sphere_id_l[write_idx]); + } + if (ii.min.x == MAX_USHORT) + // This is an invalid sphere (out of image). These spheres have + // maximum depth. Since we ordered the spheres by earliest possible + // intersection depth we re certain that there will no other sphere + // that is relevant after this one. + loading_done = true; + } + // Reset n_pixels_done. + n_pixels_done = 0; + thread_block.sync(); // Make sure n_loaded is updated. + if (n_loaded > RENDER_BUFFER_LOAD_THRESH) { + // The load buffer is full enough. Draw. + if (thread_block.thread_rank() == 0) + n_balls_loaded += n_loaded; + max_closest_possible_intersection = 0.f; + // This excludes threads outside of the image boundary. Also, it reduces + // block artifacts. + if (!done) { + for (uint draw_idx = 0; draw_idx < n_loaded; ++draw_idx) { + intersection_depth = 0.f; + if (cam_norm.orthogonal_projection) { + // The closest possible intersection is the distance to the camera + // plane. + closest_possible_intersection = min_depth_d[sphere_id_l[draw_idx]]; + } else { + closest_possible_intersection = + di_l[draw_idx].t_center - di_l[draw_idx].radius; + } + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_RENDER_PIX, + "render|drawing sphere %u (depth: %f, " + "closest possible intersection: %f).\n", + sphere_id_l[draw_idx], + di_l[draw_idx].t_center, + closest_possible_intersection); + hit = draw( + di_l[draw_idx], // Sphere to draw. + op_d == NULL ? 1.f : op_d[sphere_id_l[draw_idx]], // Opacity. + cam_norm, // Cam. + gamma, // Gamma. + ray_dir_norm, // Ray direction. + projected_ray, // Ray intersection with the image. + // Mode switches. + true, // Draw. + false, + false, + false, + false, + false, // No gradients. + // Position info. + coord_x, + coord_y, + sphere_id_l[draw_idx], + // Optional in variables. + NULL, // intersect information. + NULL, // ray_dir. + NULL, // norm_ray_dir. + NULL, // grad_pix. + &ln_pad_over_1minuspad, + // in/out variables + &sm_d, + &sm_m, + result, + // Optional out. + &depth_threshold, + &intersection_depth, + NULL, + NULL, + NULL, + NULL, + NULL // gradients. + ); + if (hit) { + max_closest_possible_intersection_hit = FMAX( + max_closest_possible_intersection_hit, + closest_possible_intersection); + tracker.track( + sphere_id_l[draw_idx], intersection_depth, coord_x, coord_y); + } + max_closest_possible_intersection = FMAX( + max_closest_possible_intersection, closest_possible_intersection); + } + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_RENDER_PIX, + "render|max_closest_possible_intersection: %f, " + "depth_threshold: %f.\n", + max_closest_possible_intersection, + depth_threshold); + } + done = done || + (percent_allowed_difference > 0.f && + max_closest_possible_intersection > depth_threshold) || + tracker.get_n_hits() >= max_n_hits; + uint warp_done = thread_warp.ballot(done); + if (thread_warp.thread_rank() == 0) + ATOMICADD_B(&n_pixels_done, POPC(warp_done)); + // This sync is necessary to keep n_loaded until all threads are done with + // painting. + thread_block.sync(); + n_loaded = 0; + } + thread_block.sync(); + } + if (thread_block.thread_rank() == 0) + n_balls_loaded += n_loaded; + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_RENDER_PIX, + "render|loaded %d balls in total.\n", + n_balls_loaded); + if (!done) { + for (uint draw_idx = 0; draw_idx < n_loaded; ++draw_idx) { + intersection_depth = 0.f; + if (cam_norm.orthogonal_projection) { + // The closest possible intersection is the distance to the camera + // plane. + closest_possible_intersection = min_depth_d[sphere_id_l[draw_idx]]; + } else { + closest_possible_intersection = + di_l[draw_idx].t_center - di_l[draw_idx].radius; + } + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_RENDER_PIX, + "render|drawing sphere %u (depth: %f, " + "closest possible intersection: %f).\n", + sphere_id_l[draw_idx], + di_l[draw_idx].t_center, + closest_possible_intersection); + hit = draw( + di_l[draw_idx], // Sphere to draw. + op_d == NULL ? 1.f : op_d[sphere_id_l[draw_idx]], // Opacity. + cam_norm, // Cam. + gamma, // Gamma. + ray_dir_norm, // Ray direction. + projected_ray, // Ray intersection with the image. + // Mode switches. + true, // Draw. + false, + false, + false, + false, + false, // No gradients. + // Logging info. + coord_x, + coord_y, + sphere_id_l[draw_idx], + // Optional in variables. + NULL, // intersect information. + NULL, // ray_dir. + NULL, // norm_ray_dir. + NULL, // grad_pix. + &ln_pad_over_1minuspad, + // in/out variables + &sm_d, + &sm_m, + result, + // Optional out. + &depth_threshold, + &intersection_depth, + NULL, + NULL, + NULL, + NULL, + NULL // gradients. + ); + if (hit) { + max_closest_possible_intersection_hit = FMAX( + max_closest_possible_intersection_hit, + closest_possible_intersection); + tracker.track( + sphere_id_l[draw_idx], intersection_depth, coord_x, coord_y); + } + } + } + if (coord_x < cam_norm.film_border_left || + coord_y < cam_norm.film_border_top || + coord_x >= cam_norm.film_border_left + cam_norm.film_width || + coord_y >= cam_norm.film_border_top + cam_norm.film_height) { + RETURN_PARALLEL(); + } + if (mode == 1u) { + // The subtractions, for example coord_y - cam_norm.film_border_left, are + // safe even though both components are uints. We checked their relation + // just above. + result_d + [(coord_y - cam_norm.film_border_top) * cam_norm.film_width * + cam_norm.n_channels + + (coord_x - cam_norm.film_border_left) * cam_norm.n_channels] = + static_cast(tracker.get_n_hits()); + } else { + float sm_d_normfac = FRCP(FMAX(sm_d, FEPS)); + for (uint c_id = 0; c_id < cam_norm.n_channels; ++c_id) + result[c_id] *= sm_d_normfac; + int write_loc = (coord_y - cam_norm.film_border_top) * cam_norm.film_width * + (3 + 2 * n_track) + + (coord_x - cam_norm.film_border_left) * (3 + 2 * n_track); + forw_info_d[write_loc] = sm_m; + forw_info_d[write_loc + 1] = sm_d; + forw_info_d[write_loc + 2] = max_closest_possible_intersection_hit; + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_RENDER_PIX, + "render|writing the %d most important ball infos.\n", + IMIN(n_track, tracker.get_n_hits())); + for (int i = 0; i < n_track; ++i) { + int sphere_id = tracker.get_closest_sphere_id(i); + IASF(sphere_id, forw_info_d[write_loc + 3 + i * 2]); + forw_info_d[write_loc + 3 + i * 2 + 1] = + tracker.get_closest_sphere_depth(i) == MAX_FLOAT + ? -1.f + : tracker.get_closest_sphere_depth(i); + PULSAR_LOG_DEV_PIX( + PULSAR_LOG_RENDER_PIX, + "render|writing %d most important: id: %d, normalized depth: %f.\n", + i, + tracker.get_closest_sphere_id(i), + tracker.get_closest_sphere_depth(i)); + } + } + END_PARALLEL_2D(); +} + +} // namespace Renderer +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.render.instantiate.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.render.instantiate.h new file mode 100644 index 0000000000000000000000000000000000000000..9c1f326e63b8b4860137d9f0d0f440896adb2a88 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/include/renderer.render.instantiate.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_INCLUDE_RENDERER_RENDER_INSTANTIATE_H_ +#define PULSAR_NATIVE_INCLUDE_RENDERER_RENDER_INSTANTIATE_H_ + +#include "./renderer.render.device.h" + +namespace pulsar { +namespace Renderer { +template GLOBAL void render( + size_t const* const RESTRICT + num_balls, /** Number of balls relevant for this pass. */ + IntersectInfo const* const RESTRICT ii_d, /** Intersect information. */ + DrawInfo const* const RESTRICT di_d, /** Draw information. */ + float const* const RESTRICT min_depth_d, /** Minimum depth per sphere. */ + int const* const RESTRICT id_d, /** IDs. */ + float const* const RESTRICT op_d, /** Opacity. */ + const CamInfo cam_norm, /** Camera normalized with all vectors to be in the + * camera coordinate system. + */ + const float gamma, /** Transparency parameter. **/ + const float percent_allowed_difference, /** Maximum allowed + error in color. */ + const uint max_n_hits, + const float* bg_col_d, + const uint mode, + const int x_min, + const int y_min, + const int x_step, + const int y_step, + // Out variables. + float* const RESTRICT result_d, /** The result image. */ + float* const RESTRICT forw_info_d, /** Additional information needed for the + grad computation. */ + const int n_track /** The number of spheres to track for backprop. */ +); +} +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/logging.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/logging.h new file mode 100644 index 0000000000000000000000000000000000000000..63d472257671287156ccf77531c6897beff1fcd2 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/logging.h @@ -0,0 +1,115 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_LOGGING_H_ +#define PULSAR_LOGGING_H_ + +// #define PULSAR_LOGGING_ENABLED +/** + * Enable detailed per-operation timings. + * + * This timing scheme is not appropriate to measure batched calculations. + * Use `PULSAR_TIMINGS_BATCHED_ENABLED` for that. + */ +// #define PULSAR_TIMINGS_ENABLED +/** + * Time batched operations. + */ +// #define PULSAR_TIMINGS_BATCHED_ENABLED +#if defined(PULSAR_TIMINGS_BATCHED_ENABLED) && defined(PULSAR_TIMINGS_ENABLED) +#pragma message("Pulsar|batched and unbatched timings enabled. This will not") +#pragma message("Pulsar|create meaningful results.") +#endif + +#ifdef PULSAR_LOGGING_ENABLED + +// Control logging. +// 0: INFO, 1: WARNING, 2: ERROR, 3: FATAL (Abort after logging). +#define CAFFE2_LOG_THRESHOLD 0 +#define PULSAR_LOG_INIT false +#define PULSAR_LOG_FORWARD false +#define PULSAR_LOG_CALC_SIGNATURE false +#define PULSAR_LOG_RENDER false +#define PULSAR_LOG_RENDER_PIX false +#define PULSAR_LOG_RENDER_PIX_X 428 +#define PULSAR_LOG_RENDER_PIX_Y 669 +#define PULSAR_LOG_RENDER_PIX_ALL false +#define PULSAR_LOG_TRACKER_PIX false +#define PULSAR_LOG_TRACKER_PIX_X 428 +#define PULSAR_LOG_TRACKER_PIX_Y 669 +#define PULSAR_LOG_TRACKER_PIX_ALL false +#define PULSAR_LOG_DRAW_PIX false +#define PULSAR_LOG_DRAW_PIX_X 428 +#define PULSAR_LOG_DRAW_PIX_Y 669 +#define PULSAR_LOG_DRAW_PIX_ALL false +#define PULSAR_LOG_BACKWARD false +#define PULSAR_LOG_GRAD false +#define PULSAR_LOG_GRAD_X 509 +#define PULSAR_LOG_GRAD_Y 489 +#define PULSAR_LOG_GRAD_ALL false +#define PULSAR_LOG_NORMALIZE false +#define PULSAR_LOG_NORMALIZE_X 0 +#define PULSAR_LOG_NORMALIZE_ALL false + +#define PULSAR_LOG_DEV(ID, ...) \ + if ((ID)) { \ + printf(__VA_ARGS__); \ + } +#define PULSAR_LOG_DEV_APIX(ID, MSG, ...) \ + if ((ID) && (film_coord_x == (ID##_X) && film_coord_y == (ID##_Y)) || \ + ID##_ALL) { \ + printf( \ + "%u %u (ap %u %u)|" MSG, \ + film_coord_x, \ + film_coord_y, \ + ap_coord_x, \ + ap_coord_y, \ + __VA_ARGS__); \ + } +#define PULSAR_LOG_DEV_PIX(ID, MSG, ...) \ + if ((ID) && (coord_x == (ID##_X) && coord_y == (ID##_Y)) || ID##_ALL) { \ + printf("%u %u|" MSG, coord_x, coord_y, __VA_ARGS__); \ + } +#ifdef __CUDACC__ +#define PULSAR_LOG_DEV_PIXB(ID, MSG, ...) \ + if ((ID) && static_cast(block_area.min.x) <= (ID##_X) && \ + static_cast(block_area.max.x) > (ID##_X) && \ + static_cast(block_area.min.y) <= (ID##_Y) && \ + static_cast(block_area.max.y) > (ID##_Y)) { \ + printf("%u %u|" MSG, coord_x, coord_y, __VA_ARGS__); \ + } +#else +#define PULSAR_LOG_DEV_PIXB(ID, MSG, ...) \ + if ((ID) && coord_x == (ID##_X) && coord_y == (ID##_Y)) { \ + printf("%u %u|" MSG, coord_x, coord_y, __VA_ARGS__); \ + } +#endif +#define PULSAR_LOG_DEV_NODE(ID, MSG, ...) \ + if ((ID) && idx == (ID##_X) || (ID##_ALL)) { \ + printf("%u|" MSG, idx, __VA_ARGS__); \ + } + +#else + +#define CAFFE2_LOG_THRESHOLD 2 + +#define PULSAR_LOG_RENDER false +#define PULSAR_LOG_INIT false +#define PULSAR_LOG_FORWARD false +#define PULSAR_LOG_BACKWARD false +#define PULSAR_LOG_TRACKER_PIX false + +#define PULSAR_LOG_DEV(...) +#define PULSAR_LOG_DEV_APIX(...) +#define PULSAR_LOG_DEV_PIX(...) +#define PULSAR_LOG_DEV_PIXB(...) +#define PULSAR_LOG_DEV_NODE(...) + +#endif + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/pytorch/camera.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/pytorch/camera.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c3794e7edf90f4af50632ea91bc131bd87fd751f --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/pytorch/camera.cpp @@ -0,0 +1,70 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "./camera.h" +#include "../include/math.h" + +namespace pulsar { +namespace pytorch { + +CamInfo cam_info_from_params( + const torch::Tensor& cam_pos, + const torch::Tensor& pixel_0_0_center, + const torch::Tensor& pixel_vec_x, + const torch::Tensor& pixel_vec_y, + const torch::Tensor& principal_point_offset, + const float& focal_length, + const uint& width, + const uint& height, + const float& min_dist, + const float& max_dist, + const bool& right_handed) { + CamInfo res; + fill_cam_vecs( + cam_pos.detach().cpu(), + pixel_0_0_center.detach().cpu(), + pixel_vec_x.detach().cpu(), + pixel_vec_y.detach().cpu(), + principal_point_offset.detach().cpu(), + right_handed, + &res); + res.half_pixel_size = 0.5f * length(res.pixel_dir_x); + if (length(res.pixel_dir_y) * 0.5f - res.half_pixel_size > EPS) { + throw std::runtime_error("Pixel sizes must agree in x and y direction!"); + } + res.focal_length = focal_length; + res.aperture_width = + width + 2u * static_cast(abs(res.principal_point_offset_x)); + res.aperture_height = + height + 2u * static_cast(abs(res.principal_point_offset_y)); + res.pixel_0_0_center -= + res.pixel_dir_x * static_cast(abs(res.principal_point_offset_x)); + res.pixel_0_0_center -= + res.pixel_dir_y * static_cast(abs(res.principal_point_offset_y)); + res.film_width = width; + res.film_height = height; + res.film_border_left = + static_cast(std::max(0, 2 * res.principal_point_offset_x)); + res.film_border_top = + static_cast(std::max(0, 2 * res.principal_point_offset_y)); + LOG_IF(INFO, PULSAR_LOG_INIT) + << "Aperture width, height: " << res.aperture_width << ", " + << res.aperture_height; + LOG_IF(INFO, PULSAR_LOG_INIT) + << "Film width, height: " << res.film_width << ", " << res.film_height; + LOG_IF(INFO, PULSAR_LOG_INIT) + << "Film border left, top: " << res.film_border_left << ", " + << res.film_border_top; + res.min_dist = min_dist; + res.max_dist = max_dist; + res.norm_fac = 1.f / (max_dist - min_dist); + return res; +}; + +} // namespace pytorch +} // namespace pulsar diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/pytorch/camera.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/pytorch/camera.h new file mode 100644 index 0000000000000000000000000000000000000000..9ecd95353ad76efd2760a4a634493917fda7b468 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/pytorch/camera.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_CAMERA_H_ +#define PULSAR_NATIVE_CAMERA_H_ + +#include +#include "../global.h" + +#include "../include/camera.h" + +namespace pulsar { +namespace pytorch { + +inline void fill_cam_vecs( + const torch::Tensor& pos_vec, + const torch::Tensor& pixel_0_0_center, + const torch::Tensor& pixel_dir_x, + const torch::Tensor& pixel_dir_y, + const torch::Tensor& principal_point_offset, + const bool& right_handed, + CamInfo* res) { + res->eye.x = pos_vec.data_ptr()[0]; + res->eye.y = pos_vec.data_ptr()[1]; + res->eye.z = pos_vec.data_ptr()[2]; + res->pixel_0_0_center.x = pixel_0_0_center.data_ptr()[0]; + res->pixel_0_0_center.y = pixel_0_0_center.data_ptr()[1]; + res->pixel_0_0_center.z = pixel_0_0_center.data_ptr()[2]; + res->pixel_dir_x.x = pixel_dir_x.data_ptr()[0]; + res->pixel_dir_x.y = pixel_dir_x.data_ptr()[1]; + res->pixel_dir_x.z = pixel_dir_x.data_ptr()[2]; + res->pixel_dir_y.x = pixel_dir_y.data_ptr()[0]; + res->pixel_dir_y.y = pixel_dir_y.data_ptr()[1]; + res->pixel_dir_y.z = pixel_dir_y.data_ptr()[2]; + auto sensor_dir_z = pixel_dir_y.cross(pixel_dir_x, -1); + sensor_dir_z /= sensor_dir_z.norm(); + if (right_handed) { + sensor_dir_z *= -1.f; + } + res->sensor_dir_z.x = sensor_dir_z.data_ptr()[0]; + res->sensor_dir_z.y = sensor_dir_z.data_ptr()[1]; + res->sensor_dir_z.z = sensor_dir_z.data_ptr()[2]; + res->principal_point_offset_x = principal_point_offset.data_ptr()[0]; + res->principal_point_offset_y = principal_point_offset.data_ptr()[1]; +} + +CamInfo cam_info_from_params( + const torch::Tensor& cam_pos, + const torch::Tensor& pixel_0_0_center, + const torch::Tensor& pixel_vec_x, + const torch::Tensor& pixel_vec_y, + const torch::Tensor& principal_point_offset, + const float& focal_length, + const uint& width, + const uint& height, + const float& min_dist, + const float& max_dist, + const bool& right_handed); + +} // namespace pytorch +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/pytorch/renderer.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/pytorch/renderer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4349aea796cfea0a63a5f76f7669816993fe3f2b --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/pytorch/renderer.cpp @@ -0,0 +1,1599 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "./renderer.h" +#include "../include/commands.h" +#include "./camera.h" +#include "./util.h" + +#include +#ifdef WITH_CUDA +#include +#include +#endif + +#ifndef TORCH_CHECK_ARG +// torch <= 1.10 +#define TORCH_CHECK_ARG(cond, argN, ...) \ + TORCH_CHECK(cond, "invalid argument ", argN, ": ", __VA_ARGS__) +#endif + +namespace PRE = ::pulsar::Renderer; + +namespace pulsar { +namespace pytorch { + +Renderer::Renderer( + const unsigned int& width, + const unsigned int& height, + const unsigned int& max_n_balls, + const bool& orthogonal_projection, + const bool& right_handed_system, + const float& background_normalization_depth, + const uint& n_channels, + const uint& n_track) { + LOG_IF(INFO, PULSAR_LOG_INIT) << "Initializing renderer."; + TORCH_CHECK_ARG(width > 0, 1, "image width must be > 0!"); + TORCH_CHECK_ARG(height > 0, 2, "image height must be > 0!"); + TORCH_CHECK_ARG(max_n_balls > 0, 3, "max_n_balls must be > 0!"); + TORCH_CHECK_ARG( + background_normalization_depth > 0.f && + background_normalization_depth < 1.f, + 5, + "background_normalization_depth must be in ]0., 1.["); + TORCH_CHECK_ARG(n_channels > 0, 6, "n_channels must be > 0"); + TORCH_CHECK_ARG( + n_track > 0 && n_track <= MAX_GRAD_SPHERES, + 7, + ("n_track must be > 0 and <" + std::to_string(MAX_GRAD_SPHERES) + + ". Is " + std::to_string(n_track) + ".") + .c_str()); + LOG_IF(INFO, PULSAR_LOG_INIT) + << "Image width: " << width << ", height: " << height; + this->renderer_vec.emplace_back(); + this->device_type = c10::DeviceType::CPU; + this->device_index = -1; + PRE::construct( + this->renderer_vec.data(), + max_n_balls, + width, + height, + orthogonal_projection, + right_handed_system, + background_normalization_depth, + n_channels, + n_track); + this->device_tracker = torch::zeros(1); +}; + +Renderer::~Renderer() { + if (this->device_type == c10::DeviceType::CUDA) { +// Can't happen in the case that not compiled with CUDA. +#ifdef WITH_CUDA + at::cuda::CUDAGuard device_guard(this->device_tracker.device()); + for (auto nrend : this->renderer_vec) { + PRE::destruct(&nrend); + } +#endif + } else { + for (auto nrend : this->renderer_vec) { + PRE::destruct(&nrend); + } + } +} + +bool Renderer::operator==(const Renderer& rhs) const { + LOG_IF(INFO, PULSAR_LOG_INIT) << "Equality check."; + bool renderer_agrees = (this->renderer_vec[0] == rhs.renderer_vec[0]); + LOG_IF(INFO, PULSAR_LOG_INIT) << " Renderer agrees: " << renderer_agrees; + bool device_agrees = + (this->device_tracker.device() == rhs.device_tracker.device()); + LOG_IF(INFO, PULSAR_LOG_INIT) << " Device agrees: " << device_agrees; + return (renderer_agrees && device_agrees); +}; + +void Renderer::ensure_on_device(torch::Device device, bool /*non_blocking*/) { + TORCH_CHECK_ARG( + device.type() == c10::DeviceType::CUDA || + device.type() == c10::DeviceType::CPU, + 1, + "Only CPU and CUDA device types are supported."); + if (device.type() != this->device_type || + device.index() != this->device_index) { +#ifdef WITH_CUDA + LOG_IF(INFO, PULSAR_LOG_INIT) + << "Transferring render buffers between devices."; + int prev_active; + cudaGetDevice(&prev_active); + if (this->device_type == c10::DeviceType::CUDA) { + LOG_IF(INFO, PULSAR_LOG_INIT) << " Destructing on CUDA."; + cudaSetDevice(this->device_index); + for (auto& nrend : this->renderer_vec) { + PRE::destruct(&nrend); + } + } else { + LOG_IF(INFO, PULSAR_LOG_INIT) << " Destructing on CPU."; + for (auto& nrend : this->renderer_vec) { + PRE::destruct(&nrend); + } + } + if (device.type() == c10::DeviceType::CUDA) { + LOG_IF(INFO, PULSAR_LOG_INIT) << " Constructing on CUDA."; + cudaSetDevice(device.index()); + for (auto& nrend : this->renderer_vec) { + PRE::construct( + &nrend, + this->renderer_vec[0].max_num_balls, + this->renderer_vec[0].cam.film_width, + this->renderer_vec[0].cam.film_height, + this->renderer_vec[0].cam.orthogonal_projection, + this->renderer_vec[0].cam.right_handed, + this->renderer_vec[0].cam.background_normalization_depth, + this->renderer_vec[0].cam.n_channels, + this->n_track()); + } + } else { + LOG_IF(INFO, PULSAR_LOG_INIT) << " Constructing on CPU."; + for (auto& nrend : this->renderer_vec) { + PRE::construct( + &nrend, + this->renderer_vec[0].max_num_balls, + this->renderer_vec[0].cam.film_width, + this->renderer_vec[0].cam.film_height, + this->renderer_vec[0].cam.orthogonal_projection, + this->renderer_vec[0].cam.right_handed, + this->renderer_vec[0].cam.background_normalization_depth, + this->renderer_vec[0].cam.n_channels, + this->n_track()); + } + } + cudaSetDevice(prev_active); + this->device_type = device.type(); + this->device_index = device.index(); +#else + throw std::runtime_error( + "pulsar was built without CUDA " + "but a device move to a CUDA device was initiated."); +#endif + } +}; + +void Renderer::ensure_n_renderers_gte(const size_t& batch_size) { + if (this->renderer_vec.size() < batch_size) { + ptrdiff_t diff = batch_size - this->renderer_vec.size(); + LOG_IF(INFO, PULSAR_LOG_INIT) + << "Increasing render buffers by " << diff + << " to account for batch size " << batch_size; + for (ptrdiff_t i = 0; i < diff; ++i) { + this->renderer_vec.emplace_back(); + if (this->device_type == c10::DeviceType::CUDA) { +#ifdef WITH_CUDA + PRE::construct( + &this->renderer_vec[this->renderer_vec.size() - 1], + this->max_num_balls(), + this->width(), + this->height(), + this->renderer_vec[0].cam.orthogonal_projection, + this->renderer_vec[0].cam.right_handed, + this->renderer_vec[0].cam.background_normalization_depth, + this->renderer_vec[0].cam.n_channels, + this->n_track()); +#endif + } else { + PRE::construct( + &this->renderer_vec[this->renderer_vec.size() - 1], + this->max_num_balls(), + this->width(), + this->height(), + this->renderer_vec[0].cam.orthogonal_projection, + this->renderer_vec[0].cam.right_handed, + this->renderer_vec[0].cam.background_normalization_depth, + this->renderer_vec[0].cam.n_channels, + this->n_track()); + } + } + } +} + +std::tuple Renderer::arg_check( + const torch::Tensor& vert_pos, + const torch::Tensor& vert_col, + const torch::Tensor& vert_radii, + const torch::Tensor& cam_pos, + const torch::Tensor& pixel_0_0_center, + const torch::Tensor& pixel_vec_x, + const torch::Tensor& pixel_vec_y, + const torch::Tensor& focal_length, + const torch::Tensor& principal_point_offsets, + const float& gamma, + const float& max_depth, + float& min_depth, + const c10::optional& bg_col, + const c10::optional& opacity, + const float& percent_allowed_difference, + const uint& max_n_hits, + const uint& mode) { + LOG_IF(INFO, PULSAR_LOG_FORWARD || PULSAR_LOG_BACKWARD) << "Arg check."; + size_t batch_size = 1; + size_t n_points; + bool batch_processing = false; + if (vert_pos.ndimension() == 3) { + // Check all parameters adhere batch size. + batch_processing = true; + batch_size = vert_pos.size(0); + TORCH_CHECK_ARG( + vert_col.ndimension() == 3 && + vert_col.size(0) == static_cast(batch_size), + 2, + "vert_col needs to have batch size."); + TORCH_CHECK_ARG( + vert_radii.ndimension() == 2 && + vert_radii.size(0) == static_cast(batch_size), + 3, + "vert_radii must be specified per batch."); + TORCH_CHECK_ARG( + cam_pos.ndimension() == 2 && + cam_pos.size(0) == static_cast(batch_size), + 4, + "cam_pos must be specified per batch and have the correct batch size."); + TORCH_CHECK_ARG( + pixel_0_0_center.ndimension() == 2 && + pixel_0_0_center.size(0) == static_cast(batch_size), + 5, + "pixel_0_0_center must be specified per batch."); + TORCH_CHECK_ARG( + pixel_vec_x.ndimension() == 2 && + pixel_vec_x.size(0) == static_cast(batch_size), + 6, + "pixel_vec_x must be specified per batch."); + TORCH_CHECK_ARG( + pixel_vec_y.ndimension() == 2 && + pixel_vec_y.size(0) == static_cast(batch_size), + 7, + "pixel_vec_y must be specified per batch."); + TORCH_CHECK_ARG( + focal_length.ndimension() == 1 && + focal_length.size(0) == static_cast(batch_size), + 8, + "focal_length must be specified per batch."); + TORCH_CHECK_ARG( + principal_point_offsets.ndimension() == 2 && + principal_point_offsets.size(0) == static_cast(batch_size), + 9, + "principal_point_offsets must be specified per batch."); + if (opacity.has_value()) { + TORCH_CHECK_ARG( + opacity.value().ndimension() == 2 && + opacity.value().size(0) == static_cast(batch_size), + 13, + "Opacity needs to be specified batch-wise."); + } + // Check all parameters are for a matching number of points. + n_points = vert_pos.size(1); + TORCH_CHECK_ARG( + vert_col.size(1) == static_cast(n_points), + 2, + ("The number of points for vertex positions (" + + std::to_string(n_points) + ") and vertex colors (" + + std::to_string(vert_col.size(1)) + ") doesn't agree.") + .c_str()); + TORCH_CHECK_ARG( + vert_radii.size(1) == static_cast(n_points), + 3, + ("The number of points for vertex positions (" + + std::to_string(n_points) + ") and vertex radii (" + + std::to_string(vert_col.size(1)) + ") doesn't agree.") + .c_str()); + if (opacity.has_value()) { + TORCH_CHECK_ARG( + opacity.value().size(1) == static_cast(n_points), + 13, + "Opacity needs to be specified per point."); + } + // Check all parameters have the correct last dimension size. + TORCH_CHECK_ARG( + vert_pos.size(2) == 3, + 1, + ("Vertex positions must be 3D (have shape " + + std::to_string(vert_pos.size(2)) + ")!") + .c_str()); + TORCH_CHECK_ARG( + vert_col.size(2) == this->renderer_vec[0].cam.n_channels, + 2, + ("Vertex colors must have the right number of channels (have shape " + + std::to_string(vert_col.size(2)) + ", need " + + std::to_string(this->renderer_vec[0].cam.n_channels) + ")!") + .c_str()); + TORCH_CHECK_ARG( + cam_pos.size(1) == 3, + 4, + ("Camera position must be 3D (has shape " + + std::to_string(cam_pos.size(1)) + ")!") + .c_str()); + TORCH_CHECK_ARG( + pixel_0_0_center.size(1) == 3, + 5, + ("pixel_0_0_center must be 3D (has shape " + + std::to_string(pixel_0_0_center.size(1)) + ")!") + .c_str()); + TORCH_CHECK_ARG( + pixel_vec_x.size(1) == 3, + 6, + ("pixel_vec_x must be 3D (has shape " + + std::to_string(pixel_vec_x.size(1)) + ")!") + .c_str()); + TORCH_CHECK_ARG( + pixel_vec_y.size(1) == 3, + 7, + ("pixel_vec_y must be 3D (has shape " + + std::to_string(pixel_vec_y.size(1)) + ")!") + .c_str()); + TORCH_CHECK_ARG( + principal_point_offsets.size(1) == 2, + 9, + "principal_point_offsets must contain x and y offsets."); + // Ensure enough renderers are available for the batch. + ensure_n_renderers_gte(batch_size); + } else { + // Check all parameters are of correct dimension. + TORCH_CHECK_ARG( + vert_col.ndimension() == 2, 2, "vert_col needs to have dimension 2."); + TORCH_CHECK_ARG( + vert_radii.ndimension() == 1, 3, "vert_radii must have dimension 1."); + TORCH_CHECK_ARG( + cam_pos.ndimension() == 1, 4, "cam_pos must have dimension 1."); + TORCH_CHECK_ARG( + pixel_0_0_center.ndimension() == 1, + 5, + "pixel_0_0_center must have dimension 1."); + TORCH_CHECK_ARG( + pixel_vec_x.ndimension() == 1, 6, "pixel_vec_x must have dimension 1."); + TORCH_CHECK_ARG( + pixel_vec_y.ndimension() == 1, 7, "pixel_vec_y must have dimension 1."); + TORCH_CHECK_ARG( + focal_length.ndimension() == 0, + 8, + "focal_length must have dimension 0."); + TORCH_CHECK_ARG( + principal_point_offsets.ndimension() == 1, + 9, + "principal_point_offsets must have dimension 1."); + if (opacity.has_value()) { + TORCH_CHECK_ARG( + opacity.value().ndimension() == 1, + 13, + "Opacity needs to be specified per sample."); + } + // Check each. + n_points = vert_pos.size(0); + TORCH_CHECK_ARG( + vert_col.size(0) == static_cast(n_points), + 2, + ("The number of points for vertex positions (" + + std::to_string(n_points) + ") and vertex colors (" + + std::to_string(vert_col.size(0)) + ") doesn't agree.") + .c_str()); + TORCH_CHECK_ARG( + vert_radii.size(0) == static_cast(n_points), + 3, + ("The number of points for vertex positions (" + + std::to_string(n_points) + ") and vertex radii (" + + std::to_string(vert_col.size(0)) + ") doesn't agree.") + .c_str()); + if (opacity.has_value()) { + TORCH_CHECK_ARG( + opacity.value().size(0) == static_cast(n_points), + 12, + "Opacity needs to be specified per point."); + } + // Check all parameters have the correct last dimension size. + TORCH_CHECK_ARG( + vert_pos.size(1) == 3, + 1, + ("Vertex positions must be 3D (have shape " + + std::to_string(vert_pos.size(1)) + ")!") + .c_str()); + TORCH_CHECK_ARG( + vert_col.size(1) == this->renderer_vec[0].cam.n_channels, + 2, + ("Vertex colors must have the right number of channels (have shape " + + std::to_string(vert_col.size(1)) + ", need " + + std::to_string(this->renderer_vec[0].cam.n_channels) + ")!") + .c_str()); + TORCH_CHECK_ARG( + cam_pos.size(0) == 3, + 4, + ("Camera position must be 3D (has shape " + + std::to_string(cam_pos.size(0)) + ")!") + .c_str()); + TORCH_CHECK_ARG( + pixel_0_0_center.size(0) == 3, + 5, + ("pixel_0_0_center must be 3D (has shape " + + std::to_string(pixel_0_0_center.size(0)) + ")!") + .c_str()); + TORCH_CHECK_ARG( + pixel_vec_x.size(0) == 3, + 6, + ("pixel_vec_x must be 3D (has shape " + + std::to_string(pixel_vec_x.size(0)) + ")!") + .c_str()); + TORCH_CHECK_ARG( + pixel_vec_y.size(0) == 3, + 7, + ("pixel_vec_y must be 3D (has shape " + + std::to_string(pixel_vec_y.size(0)) + ")!") + .c_str()); + TORCH_CHECK_ARG( + principal_point_offsets.size(0) == 2, + 9, + "principal_point_offsets must have x and y component."); + } + // Check device placement. + auto dev = torch::device_of(vert_pos).value(); + TORCH_CHECK_ARG( + dev.type() == this->device_type && dev.index() == this->device_index, + 1, + ("Vertex positions must be stored on device " + + c10::DeviceTypeName(this->device_type) + ", index " + + std::to_string(this->device_index) + "! Are stored on " + + c10::DeviceTypeName(dev.type()) + ", index " + + std::to_string(dev.index()) + ".") + .c_str()); + dev = torch::device_of(vert_col).value(); + TORCH_CHECK_ARG( + dev.type() == this->device_type && dev.index() == this->device_index, + 2, + ("Vertex colors must be stored on device " + + c10::DeviceTypeName(this->device_type) + ", index " + + std::to_string(this->device_index) + "! Are stored on " + + c10::DeviceTypeName(dev.type()) + ", index " + + std::to_string(dev.index()) + ".") + .c_str()); + dev = torch::device_of(vert_radii).value(); + TORCH_CHECK_ARG( + dev.type() == this->device_type && dev.index() == this->device_index, + 3, + ("Vertex radii must be stored on device " + + c10::DeviceTypeName(this->device_type) + ", index " + + std::to_string(this->device_index) + "! Are stored on " + + c10::DeviceTypeName(dev.type()) + ", index " + + std::to_string(dev.index()) + ".") + .c_str()); + dev = torch::device_of(cam_pos).value(); + TORCH_CHECK_ARG( + dev.type() == this->device_type && dev.index() == this->device_index, + 4, + ("Camera position must be stored on device " + + c10::DeviceTypeName(this->device_type) + ", index " + + std::to_string(this->device_index) + "! Are stored on " + + c10::DeviceTypeName(dev.type()) + ", index " + + std::to_string(dev.index()) + ".") + .c_str()); + dev = torch::device_of(pixel_0_0_center).value(); + TORCH_CHECK_ARG( + dev.type() == this->device_type && dev.index() == this->device_index, + 5, + ("pixel_0_0_center must be stored on device " + + c10::DeviceTypeName(this->device_type) + ", index " + + std::to_string(this->device_index) + "! Are stored on " + + c10::DeviceTypeName(dev.type()) + ", index " + + std::to_string(dev.index()) + ".") + .c_str()); + dev = torch::device_of(pixel_vec_x).value(); + TORCH_CHECK_ARG( + dev.type() == this->device_type && dev.index() == this->device_index, + 6, + ("pixel_vec_x must be stored on device " + + c10::DeviceTypeName(this->device_type) + ", index " + + std::to_string(this->device_index) + "! Are stored on " + + c10::DeviceTypeName(dev.type()) + ", index " + + std::to_string(dev.index()) + ".") + .c_str()); + dev = torch::device_of(pixel_vec_y).value(); + TORCH_CHECK_ARG( + dev.type() == this->device_type && dev.index() == this->device_index, + 7, + ("pixel_vec_y must be stored on device " + + c10::DeviceTypeName(this->device_type) + ", index " + + std::to_string(this->device_index) + "! Are stored on " + + c10::DeviceTypeName(dev.type()) + ", index " + + std::to_string(dev.index()) + ".") + .c_str()); + dev = torch::device_of(principal_point_offsets).value(); + TORCH_CHECK_ARG( + dev.type() == this->device_type && dev.index() == this->device_index, + 9, + ("principal_point_offsets must be stored on device " + + c10::DeviceTypeName(this->device_type) + ", index " + + std::to_string(this->device_index) + "! Are stored on " + + c10::DeviceTypeName(dev.type()) + ", index " + + std::to_string(dev.index()) + ".") + .c_str()); + if (opacity.has_value()) { + dev = torch::device_of(opacity.value()).value(); + TORCH_CHECK_ARG( + dev.type() == this->device_type && dev.index() == this->device_index, + 13, + ("opacity must be stored on device " + + c10::DeviceTypeName(this->device_type) + ", index " + + std::to_string(this->device_index) + "! Is stored on " + + c10::DeviceTypeName(dev.type()) + ", index " + + std::to_string(dev.index()) + ".") + .c_str()); + } + // Type checks. + TORCH_CHECK_ARG( + vert_pos.scalar_type() == c10::kFloat, 1, "pulsar requires float types."); + TORCH_CHECK_ARG( + vert_col.scalar_type() == c10::kFloat, 2, "pulsar requires float types."); + TORCH_CHECK_ARG( + vert_radii.scalar_type() == c10::kFloat, + 3, + "pulsar requires float types."); + TORCH_CHECK_ARG( + cam_pos.scalar_type() == c10::kFloat, 4, "pulsar requires float types."); + TORCH_CHECK_ARG( + pixel_0_0_center.scalar_type() == c10::kFloat, + 5, + "pulsar requires float types."); + TORCH_CHECK_ARG( + pixel_vec_x.scalar_type() == c10::kFloat, + 6, + "pulsar requires float types."); + TORCH_CHECK_ARG( + pixel_vec_y.scalar_type() == c10::kFloat, + 7, + "pulsar requires float types."); + TORCH_CHECK_ARG( + focal_length.scalar_type() == c10::kFloat, + 8, + "pulsar requires float types."); + TORCH_CHECK_ARG( + // Unfortunately, the PyTorch interface is inconsistent for + // Int32: in Python, there exists an explicit int32 type, in + // C++ this is currently `c10::kInt`. + principal_point_offsets.scalar_type() == c10::kInt, + 9, + "principal_point_offsets must be provided as int32."); + if (opacity.has_value()) { + TORCH_CHECK_ARG( + opacity.value().scalar_type() == c10::kFloat, + 13, + "opacity must be a float type."); + } + // Content checks. + TORCH_CHECK_ARG( + (vert_radii > FEPS).all().item(), + 3, + ("Vertex radii must be > FEPS (min is " + + std::to_string(vert_radii.min().item()) + ").") + .c_str()); + if (this->orthogonal()) { + TORCH_CHECK_ARG( + (focal_length == 0.f).all().item(), + 8, + ("for an orthogonal projection focal length must be zero (abs max: " + + std::to_string(focal_length.abs().max().item()) + ").") + .c_str()); + } else { + TORCH_CHECK_ARG( + (focal_length > FEPS).all().item(), + 8, + ("for a perspective projection focal length must be > FEPS (min " + + std::to_string(focal_length.min().item()) + ").") + .c_str()); + } + TORCH_CHECK_ARG( + gamma <= 1.f && gamma >= 1E-5f, + 10, + ("gamma must be in [1E-5, 1] (" + std::to_string(gamma) + ").").c_str()); + if (min_depth == 0.f) { + min_depth = focal_length.max().item() + 2.f * FEPS; + } + TORCH_CHECK_ARG( + min_depth > focal_length.max().item(), + 12, + ("min_depth must be > focal_length (" + std::to_string(min_depth) + + " vs. " + std::to_string(focal_length.max().item()) + ").") + .c_str()); + TORCH_CHECK_ARG( + max_depth > min_depth + FEPS, + 11, + ("max_depth must be > min_depth + FEPS (" + std::to_string(max_depth) + + " vs. " + std::to_string(min_depth + FEPS) + ").") + .c_str()); + TORCH_CHECK_ARG( + percent_allowed_difference >= 0.f && percent_allowed_difference < 1.f, + 14, + ("percent_allowed_difference must be in [0., 1.[ (" + + std::to_string(percent_allowed_difference) + ").") + .c_str()); + TORCH_CHECK_ARG(max_n_hits > 0, 14, "max_n_hits must be > 0!"); + TORCH_CHECK_ARG(mode < 2, 15, "mode must be in {0, 1}."); + torch::Tensor real_bg_col; + if (bg_col.has_value()) { + TORCH_CHECK_ARG( + bg_col.value().device().type() == this->device_type && + bg_col.value().device().index() == this->device_index, + 13, + "bg_col must be stored on the renderer device!"); + TORCH_CHECK_ARG( + bg_col.value().ndimension() == 1 && + bg_col.value().size(0) == renderer_vec[0].cam.n_channels, + 13, + "bg_col must have the same number of channels as the image,)."); + real_bg_col = bg_col.value(); + } else { + real_bg_col = torch::ones( + {renderer_vec[0].cam.n_channels}, + c10::Device(this->device_type, this->device_index)) + .to(c10::kFloat); + } + if (opacity.has_value()) { + TORCH_CHECK_ARG( + (opacity.value() >= 0.f).all().item(), + 13, + "opacity must be >= 0."); + TORCH_CHECK_ARG( + (opacity.value() <= 1.f).all().item(), + 13, + "opacity must be <= 1."); + } + LOG_IF(INFO, PULSAR_LOG_FORWARD || PULSAR_LOG_BACKWARD) + << " batch_size: " << batch_size; + LOG_IF(INFO, PULSAR_LOG_FORWARD || PULSAR_LOG_BACKWARD) + << " n_points: " << n_points; + LOG_IF(INFO, PULSAR_LOG_FORWARD || PULSAR_LOG_BACKWARD) + << " batch_processing: " << batch_processing; + return std::tuple( + batch_size, n_points, batch_processing, real_bg_col); +} + +std::tuple Renderer::forward( + const torch::Tensor& vert_pos, + const torch::Tensor& vert_col, + const torch::Tensor& vert_radii, + const torch::Tensor& cam_pos, + const torch::Tensor& pixel_0_0_center, + const torch::Tensor& pixel_vec_x, + const torch::Tensor& pixel_vec_y, + const torch::Tensor& focal_length, + const torch::Tensor& principal_point_offsets, + const float& gamma, + const float& max_depth, + float min_depth, + const c10::optional& bg_col, + const c10::optional& opacity, + const float& percent_allowed_difference, + const uint& max_n_hits, + const uint& mode) { + // Parameter checks. + this->ensure_on_device(this->device_tracker.device()); + size_t batch_size; + size_t n_points; + bool batch_processing; + torch::Tensor real_bg_col; + std::tie(batch_size, n_points, batch_processing, real_bg_col) = + this->arg_check( + vert_pos, + vert_col, + vert_radii, + cam_pos, + pixel_0_0_center, + pixel_vec_x, + pixel_vec_y, + focal_length, + principal_point_offsets, + gamma, + max_depth, + min_depth, + bg_col, + opacity, + percent_allowed_difference, + max_n_hits, + mode); + LOG_IF(INFO, PULSAR_LOG_FORWARD) << "Extracting camera objects..."; + // Create the camera information. + std::vector cam_infos(batch_size); + if (batch_processing) { + for (size_t batch_i = 0; batch_i < batch_size; ++batch_i) { + cam_infos[batch_i] = cam_info_from_params( + cam_pos[batch_i], + pixel_0_0_center[batch_i], + pixel_vec_x[batch_i], + pixel_vec_y[batch_i], + principal_point_offsets[batch_i], + focal_length[batch_i].item(), + this->renderer_vec[0].cam.film_width, + this->renderer_vec[0].cam.film_height, + min_depth, + max_depth, + this->renderer_vec[0].cam.right_handed); + } + } else { + cam_infos[0] = cam_info_from_params( + cam_pos, + pixel_0_0_center, + pixel_vec_x, + pixel_vec_y, + principal_point_offsets, + focal_length.item(), + this->renderer_vec[0].cam.film_width, + this->renderer_vec[0].cam.film_height, + min_depth, + max_depth, + this->renderer_vec[0].cam.right_handed); + } + LOG_IF(INFO, PULSAR_LOG_FORWARD) << "Processing..."; + // Let's go! + // Contiguous version of opacity, if available. We need to create this object + // in scope to keep it alive. + torch::Tensor opacity_contiguous; + float const* opacity_ptr = nullptr; + if (opacity.has_value()) { + opacity_contiguous = opacity.value().contiguous(); + opacity_ptr = opacity_contiguous.data_ptr(); + } + if (this->device_type == c10::DeviceType::CUDA) { +// No else check necessary - if not compiled with CUDA +// we can't even reach this code (the renderer can't be +// moved to a CUDA device). +#ifdef WITH_CUDA + int prev_active; + cudaGetDevice(&prev_active); + cudaSetDevice(this->device_index); +#ifdef PULSAR_TIMINGS_BATCHED_ENABLED + START_TIME_CU(batch_forward); +#endif + if (batch_processing) { + for (size_t batch_i = 0; batch_i < batch_size; ++batch_i) { + // These calls are non-blocking and just kick off the computations. + PRE::forward( + &this->renderer_vec[batch_i], + vert_pos[batch_i].contiguous().data_ptr(), + vert_col[batch_i].contiguous().data_ptr(), + vert_radii[batch_i].contiguous().data_ptr(), + cam_infos[batch_i], + gamma, + percent_allowed_difference, + max_n_hits, + real_bg_col.contiguous().data_ptr(), + opacity_ptr, + n_points, + mode, + at::cuda::getCurrentCUDAStream()); + } + } else { + PRE::forward( + this->renderer_vec.data(), + vert_pos.contiguous().data_ptr(), + vert_col.contiguous().data_ptr(), + vert_radii.contiguous().data_ptr(), + cam_infos[0], + gamma, + percent_allowed_difference, + max_n_hits, + real_bg_col.contiguous().data_ptr(), + opacity_ptr, + n_points, + mode, + at::cuda::getCurrentCUDAStream()); + } +#ifdef PULSAR_TIMINGS_BATCHED_ENABLED + STOP_TIME_CU(batch_forward); + float time_ms; + GET_TIME_CU(batch_forward, &time_ms); + std::cout << "Forward render batched time per example: " + << time_ms / static_cast(batch_size) << "ms" << std::endl; +#endif + cudaSetDevice(prev_active); +#endif + } else { +#ifdef PULSAR_TIMINGS_BATCHED_ENABLED + START_TIME(batch_forward); +#endif + if (batch_processing) { + for (size_t batch_i = 0; batch_i < batch_size; ++batch_i) { + // These calls are non-blocking and just kick off the computations. + PRE::forward( + &this->renderer_vec[batch_i], + vert_pos[batch_i].contiguous().data_ptr(), + vert_col[batch_i].contiguous().data_ptr(), + vert_radii[batch_i].contiguous().data_ptr(), + cam_infos[batch_i], + gamma, + percent_allowed_difference, + max_n_hits, + real_bg_col.contiguous().data_ptr(), + opacity_ptr, + n_points, + mode, + nullptr); + } + } else { + PRE::forward( + this->renderer_vec.data(), + vert_pos.contiguous().data_ptr(), + vert_col.contiguous().data_ptr(), + vert_radii.contiguous().data_ptr(), + cam_infos[0], + gamma, + percent_allowed_difference, + max_n_hits, + real_bg_col.contiguous().data_ptr(), + opacity_ptr, + n_points, + mode, + nullptr); + } +#ifdef PULSAR_TIMINGS_BATCHED_ENABLED + STOP_TIME(batch_forward); + float time_ms; + GET_TIME(batch_forward, &time_ms); + std::cout << "Forward render batched time per example: " + << time_ms / static_cast(batch_size) << "ms" << std::endl; +#endif + } + LOG_IF(INFO, PULSAR_LOG_FORWARD) << "Extracting results..."; + // Create the results. + std::vector results(batch_size); + std::vector forw_infos(batch_size); + for (size_t batch_i = 0; batch_i < batch_size; ++batch_i) { + results[batch_i] = from_blob( + this->renderer_vec[batch_i].result_d, + {this->renderer_vec[0].cam.film_height, + this->renderer_vec[0].cam.film_width, + this->renderer_vec[0].cam.n_channels}, + this->device_type, + this->device_index, + torch::kFloat, + this->device_type == c10::DeviceType::CUDA +#ifdef WITH_CUDA + ? at::cuda::getCurrentCUDAStream() +#else + ? (cudaStream_t) nullptr +#endif + : (cudaStream_t) nullptr); + if (mode == 1) + results[batch_i] = results[batch_i].slice(2, 0, 1, 1); + forw_infos[batch_i] = from_blob( + this->renderer_vec[batch_i].forw_info_d, + {this->renderer_vec[0].cam.film_height, + this->renderer_vec[0].cam.film_width, + 3 + 2 * this->n_track()}, + this->device_type, + this->device_index, + torch::kFloat, + this->device_type == c10::DeviceType::CUDA +#ifdef WITH_CUDA + ? at::cuda::getCurrentCUDAStream() +#else + ? (cudaStream_t) nullptr +#endif + : (cudaStream_t) nullptr); + } + LOG_IF(INFO, PULSAR_LOG_FORWARD) << "Forward render complete."; + if (batch_processing) { + return std::tuple( + torch::stack(results), torch::stack(forw_infos)); + } else { + return std::tuple(results[0], forw_infos[0]); + } +}; + +std::tuple< + at::optional, + at::optional, + at::optional, + at::optional, + at::optional, + at::optional, + at::optional, + at::optional> +Renderer::backward( + const torch::Tensor& grad_im, + const torch::Tensor& image, + const torch::Tensor& forw_info, + const torch::Tensor& vert_pos, + const torch::Tensor& vert_col, + const torch::Tensor& vert_radii, + const torch::Tensor& cam_pos, + const torch::Tensor& pixel_0_0_center, + const torch::Tensor& pixel_vec_x, + const torch::Tensor& pixel_vec_y, + const torch::Tensor& focal_length, + const torch::Tensor& principal_point_offsets, + const float& gamma, + const float& max_depth, + float min_depth, + const c10::optional& bg_col, + const c10::optional& opacity, + const float& percent_allowed_difference, + const uint& max_n_hits, + const uint& mode, + const bool& dif_pos, + const bool& dif_col, + const bool& dif_rad, + const bool& dif_cam, + const bool& dif_opy, + const at::optional>& dbg_pos) { + this->ensure_on_device(this->device_tracker.device()); + size_t batch_size; + size_t n_points; + bool batch_processing; + torch::Tensor real_bg_col; + std::tie(batch_size, n_points, batch_processing, real_bg_col) = + this->arg_check( + vert_pos, + vert_col, + vert_radii, + cam_pos, + pixel_0_0_center, + pixel_vec_x, + pixel_vec_y, + focal_length, + principal_point_offsets, + gamma, + max_depth, + min_depth, + bg_col, + opacity, + percent_allowed_difference, + max_n_hits, + mode); + // Additional checks for the gradient computation. + TORCH_CHECK_ARG( + (grad_im.ndimension() == 3 + batch_processing && + static_cast(grad_im.size(0 + batch_processing)) == + this->height() && + static_cast(grad_im.size(1 + batch_processing)) == this->width() && + static_cast(grad_im.size(2 + batch_processing)) == + this->renderer_vec[0].cam.n_channels), + 1, + "The gradient image size is not correct."); + TORCH_CHECK_ARG( + (image.ndimension() == 3 + batch_processing && + static_cast(image.size(0 + batch_processing)) == this->height() && + static_cast(image.size(1 + batch_processing)) == this->width() && + static_cast(image.size(2 + batch_processing)) == + this->renderer_vec[0].cam.n_channels), + 2, + "The result image size is not correct."); + TORCH_CHECK_ARG( + grad_im.scalar_type() == c10::kFloat, + 1, + "The gradient image must be of float type."); + TORCH_CHECK_ARG( + image.scalar_type() == c10::kFloat, + 2, + "The image must be of float type."); + if (dif_opy) { + TORCH_CHECK_ARG( + opacity.has_value(), 13, "dif_opy set requires opacity values."); + } + if (batch_processing) { + TORCH_CHECK_ARG( + grad_im.size(0) == static_cast(batch_size), + 1, + "Gradient image batch size must agree."); + TORCH_CHECK_ARG( + image.size(0) == static_cast(batch_size), + 2, + "Image batch size must agree."); + TORCH_CHECK_ARG( + forw_info.size(0) == static_cast(batch_size), + 3, + "forward info must have batch size."); + } + TORCH_CHECK_ARG( + (forw_info.ndimension() == 3 + batch_processing && + static_cast(forw_info.size(0 + batch_processing)) == + this->height() && + static_cast(forw_info.size(1 + batch_processing)) == + this->width() && + static_cast(forw_info.size(2 + batch_processing)) == + 3 + 2 * this->n_track()), + 3, + "The forward info image size is not correct."); + TORCH_CHECK_ARG( + forw_info.scalar_type() == c10::kFloat, + 3, + "The forward info must be of float type."); + // Check device. + auto dev = torch::device_of(grad_im).value(); + TORCH_CHECK_ARG( + dev.type() == this->device_type && dev.index() == this->device_index, + 1, + ("grad_im must be stored on device " + + c10::DeviceTypeName(this->device_type) + ", index " + + std::to_string(this->device_index) + "! Are stored on " + + c10::DeviceTypeName(dev.type()) + ", index " + + std::to_string(dev.index()) + ".") + .c_str()); + dev = torch::device_of(image).value(); + TORCH_CHECK_ARG( + dev.type() == this->device_type && dev.index() == this->device_index, + 2, + ("image must be stored on device " + + c10::DeviceTypeName(this->device_type) + ", index " + + std::to_string(this->device_index) + "! Are stored on " + + c10::DeviceTypeName(dev.type()) + ", index " + + std::to_string(dev.index()) + ".") + .c_str()); + dev = torch::device_of(forw_info).value(); + TORCH_CHECK_ARG( + dev.type() == this->device_type && dev.index() == this->device_index, + 3, + ("forw_info must be stored on device " + + c10::DeviceTypeName(this->device_type) + ", index " + + std::to_string(this->device_index) + "! Are stored on " + + c10::DeviceTypeName(dev.type()) + ", index " + + std::to_string(dev.index()) + ".") + .c_str()); + if (dbg_pos.has_value()) { + TORCH_CHECK_ARG( + dbg_pos.value().first < this->width() && + dbg_pos.value().second < this->height(), + 23, + "The debug position must be within image bounds."); + } + // Prepare the return value. + std::tuple< + at::optional, + at::optional, + at::optional, + at::optional, + at::optional, + at::optional, + at::optional, + at::optional> + ret; + if (mode == 1 || (!dif_pos && !dif_col && !dif_rad && !dif_cam && !dif_opy)) { + return ret; + } + // Create the camera information. + std::vector cam_infos(batch_size); + if (batch_processing) { + for (size_t batch_i = 0; batch_i < batch_size; ++batch_i) { + cam_infos[batch_i] = cam_info_from_params( + cam_pos[batch_i], + pixel_0_0_center[batch_i], + pixel_vec_x[batch_i], + pixel_vec_y[batch_i], + principal_point_offsets[batch_i], + focal_length[batch_i].item(), + this->renderer_vec[0].cam.film_width, + this->renderer_vec[0].cam.film_height, + min_depth, + max_depth, + this->renderer_vec[0].cam.right_handed); + } + } else { + cam_infos[0] = cam_info_from_params( + cam_pos, + pixel_0_0_center, + pixel_vec_x, + pixel_vec_y, + principal_point_offsets, + focal_length.item(), + this->renderer_vec[0].cam.film_width, + this->renderer_vec[0].cam.film_height, + min_depth, + max_depth, + this->renderer_vec[0].cam.right_handed); + } + // Let's go! + // Contiguous version of opacity, if available. We need to create this object + // in scope to keep it alive. + torch::Tensor opacity_contiguous; + float const* opacity_ptr = nullptr; + if (opacity.has_value()) { + opacity_contiguous = opacity.value().contiguous(); + opacity_ptr = opacity_contiguous.data_ptr(); + } + if (this->device_type == c10::DeviceType::CUDA) { +// No else check necessary - it's not possible to move +// the renderer to a CUDA device if not built with CUDA. +#ifdef WITH_CUDA + int prev_active; + cudaGetDevice(&prev_active); + cudaSetDevice(this->device_index); +#ifdef PULSAR_TIMINGS_BATCHED_ENABLED + START_TIME_CU(batch_backward); +#endif + if (batch_processing) { + for (size_t batch_i = 0; batch_i < batch_size; ++batch_i) { + // These calls are non-blocking and just kick off the computations. + if (dbg_pos.has_value()) { + PRE::backward_dbg( + &this->renderer_vec[batch_i], + grad_im[batch_i].contiguous().data_ptr(), + image[batch_i].contiguous().data_ptr(), + forw_info[batch_i].contiguous().data_ptr(), + vert_pos[batch_i].contiguous().data_ptr(), + vert_col[batch_i].contiguous().data_ptr(), + vert_radii[batch_i].contiguous().data_ptr(), + cam_infos[batch_i], + gamma, + percent_allowed_difference, + max_n_hits, + opacity_ptr, + n_points, + mode, + dif_pos, + dif_col, + dif_rad, + dif_cam, + dif_opy, + dbg_pos.value().first, + dbg_pos.value().second, + at::cuda::getCurrentCUDAStream()); + } else { + PRE::backward( + &this->renderer_vec[batch_i], + grad_im[batch_i].contiguous().data_ptr(), + image[batch_i].contiguous().data_ptr(), + forw_info[batch_i].contiguous().data_ptr(), + vert_pos[batch_i].contiguous().data_ptr(), + vert_col[batch_i].contiguous().data_ptr(), + vert_radii[batch_i].contiguous().data_ptr(), + cam_infos[batch_i], + gamma, + percent_allowed_difference, + max_n_hits, + opacity_ptr, + n_points, + mode, + dif_pos, + dif_col, + dif_rad, + dif_cam, + dif_opy, + at::cuda::getCurrentCUDAStream()); + } + } + } else { + if (dbg_pos.has_value()) { + PRE::backward_dbg( + this->renderer_vec.data(), + grad_im.contiguous().data_ptr(), + image.contiguous().data_ptr(), + forw_info.contiguous().data_ptr(), + vert_pos.contiguous().data_ptr(), + vert_col.contiguous().data_ptr(), + vert_radii.contiguous().data_ptr(), + cam_infos[0], + gamma, + percent_allowed_difference, + max_n_hits, + opacity_ptr, + n_points, + mode, + dif_pos, + dif_col, + dif_rad, + dif_cam, + dif_opy, + dbg_pos.value().first, + dbg_pos.value().second, + at::cuda::getCurrentCUDAStream()); + } else { + PRE::backward( + this->renderer_vec.data(), + grad_im.contiguous().data_ptr(), + image.contiguous().data_ptr(), + forw_info.contiguous().data_ptr(), + vert_pos.contiguous().data_ptr(), + vert_col.contiguous().data_ptr(), + vert_radii.contiguous().data_ptr(), + cam_infos[0], + gamma, + percent_allowed_difference, + max_n_hits, + opacity_ptr, + n_points, + mode, + dif_pos, + dif_col, + dif_rad, + dif_cam, + dif_opy, + at::cuda::getCurrentCUDAStream()); + } + } + cudaSetDevice(prev_active); +#ifdef PULSAR_TIMINGS_BATCHED_ENABLED + STOP_TIME_CU(batch_backward); + float time_ms; + GET_TIME_CU(batch_backward, &time_ms); + std::cout << "Backward render batched time per example: " + << time_ms / static_cast(batch_size) << "ms" << std::endl; +#endif +#endif // WITH_CUDA + } else { +#ifdef PULSAR_TIMINGS_BATCHED_ENABLED + START_TIME(batch_backward); +#endif + if (batch_processing) { + for (size_t batch_i = 0; batch_i < batch_size; ++batch_i) { + // These calls are non-blocking and just kick off the computations. + if (dbg_pos.has_value()) { + PRE::backward_dbg( + &this->renderer_vec[batch_i], + grad_im[batch_i].contiguous().data_ptr(), + image[batch_i].contiguous().data_ptr(), + forw_info[batch_i].contiguous().data_ptr(), + vert_pos[batch_i].contiguous().data_ptr(), + vert_col[batch_i].contiguous().data_ptr(), + vert_radii[batch_i].contiguous().data_ptr(), + cam_infos[batch_i], + gamma, + percent_allowed_difference, + max_n_hits, + opacity_ptr, + n_points, + mode, + dif_pos, + dif_col, + dif_rad, + dif_cam, + dif_opy, + dbg_pos.value().first, + dbg_pos.value().second, + nullptr); + } else { + PRE::backward( + &this->renderer_vec[batch_i], + grad_im[batch_i].contiguous().data_ptr(), + image[batch_i].contiguous().data_ptr(), + forw_info[batch_i].contiguous().data_ptr(), + vert_pos[batch_i].contiguous().data_ptr(), + vert_col[batch_i].contiguous().data_ptr(), + vert_radii[batch_i].contiguous().data_ptr(), + cam_infos[batch_i], + gamma, + percent_allowed_difference, + max_n_hits, + opacity_ptr, + n_points, + mode, + dif_pos, + dif_col, + dif_rad, + dif_cam, + dif_opy, + nullptr); + } + } + } else { + if (dbg_pos.has_value()) { + PRE::backward_dbg( + this->renderer_vec.data(), + grad_im.contiguous().data_ptr(), + image.contiguous().data_ptr(), + forw_info.contiguous().data_ptr(), + vert_pos.contiguous().data_ptr(), + vert_col.contiguous().data_ptr(), + vert_radii.contiguous().data_ptr(), + cam_infos[0], + gamma, + percent_allowed_difference, + max_n_hits, + opacity_ptr, + n_points, + mode, + dif_pos, + dif_col, + dif_rad, + dif_cam, + dif_opy, + dbg_pos.value().first, + dbg_pos.value().second, + nullptr); + } else { + PRE::backward( + this->renderer_vec.data(), + grad_im.contiguous().data_ptr(), + image.contiguous().data_ptr(), + forw_info.contiguous().data_ptr(), + vert_pos.contiguous().data_ptr(), + vert_col.contiguous().data_ptr(), + vert_radii.contiguous().data_ptr(), + cam_infos[0], + gamma, + percent_allowed_difference, + max_n_hits, + opacity_ptr, + n_points, + mode, + dif_pos, + dif_col, + dif_rad, + dif_cam, + dif_opy, + nullptr); + } + } +#ifdef PULSAR_TIMINGS_BATCHED_ENABLED + STOP_TIME(batch_backward); + float time_ms; + GET_TIME(batch_backward, &time_ms); + std::cout << "Backward render batched time per example: " + << time_ms / static_cast(batch_size) << "ms" << std::endl; +#endif + } + if (dif_pos) { + if (batch_processing) { + std::vector results(batch_size); + for (size_t batch_i = 0; batch_i < batch_size; ++batch_i) { + results[batch_i] = from_blob( + reinterpret_cast(this->renderer_vec[batch_i].grad_pos_d), + {static_cast(n_points), 3}, + this->device_type, + this->device_index, + torch::kFloat, + this->device_type == c10::DeviceType::CUDA +#ifdef WITH_CUDA + ? at::cuda::getCurrentCUDAStream() +#else + ? (cudaStream_t) nullptr +#endif + : (cudaStream_t) nullptr); + } + std::get<0>(ret) = torch::stack(results); + } else { + std::get<0>(ret) = from_blob( + reinterpret_cast(this->renderer_vec[0].grad_pos_d), + {static_cast(n_points), 3}, + this->device_type, + this->device_index, + torch::kFloat, + this->device_type == c10::DeviceType::CUDA +#ifdef WITH_CUDA + ? at::cuda::getCurrentCUDAStream() +#else + ? (cudaStream_t) nullptr +#endif + : (cudaStream_t) nullptr); + } + } + if (dif_col) { + if (batch_processing) { + std::vector results(batch_size); + for (size_t batch_i = 0; batch_i < batch_size; ++batch_i) { + results[batch_i] = from_blob( + reinterpret_cast(this->renderer_vec[batch_i].grad_col_d), + {static_cast(n_points), + this->renderer_vec[0].cam.n_channels}, + this->device_type, + this->device_index, + torch::kFloat, + this->device_type == c10::DeviceType::CUDA +#ifdef WITH_CUDA + ? at::cuda::getCurrentCUDAStream() +#else + ? (cudaStream_t) nullptr +#endif + : (cudaStream_t) nullptr); + } + std::get<1>(ret) = torch::stack(results); + } else { + std::get<1>(ret) = from_blob( + reinterpret_cast(this->renderer_vec[0].grad_col_d), + {static_cast(n_points), + this->renderer_vec[0].cam.n_channels}, + this->device_type, + this->device_index, + torch::kFloat, + this->device_type == c10::DeviceType::CUDA +#ifdef WITH_CUDA + ? at::cuda::getCurrentCUDAStream() +#else + ? (cudaStream_t) nullptr +#endif + : (cudaStream_t) nullptr); + } + } + if (dif_rad) { + if (batch_processing) { + std::vector results(batch_size); + for (size_t batch_i = 0; batch_i < batch_size; ++batch_i) { + results[batch_i] = from_blob( + reinterpret_cast(this->renderer_vec[batch_i].grad_rad_d), + {static_cast(n_points)}, + this->device_type, + this->device_index, + torch::kFloat, + this->device_type == c10::DeviceType::CUDA +#ifdef WITH_CUDA + ? at::cuda::getCurrentCUDAStream() +#else + ? (cudaStream_t) nullptr +#endif + : (cudaStream_t) nullptr); + } + std::get<2>(ret) = torch::stack(results); + } else { + std::get<2>(ret) = from_blob( + reinterpret_cast(this->renderer_vec[0].grad_rad_d), + {static_cast(n_points)}, + this->device_type, + this->device_index, + torch::kFloat, + this->device_type == c10::DeviceType::CUDA +#ifdef WITH_CUDA + ? at::cuda::getCurrentCUDAStream() +#else + ? (cudaStream_t) nullptr +#endif + : (cudaStream_t) nullptr); + } + } + if (dif_cam) { + if (batch_processing) { + std::vector res_p1(batch_size); + std::vector res_p2(batch_size); + std::vector res_p3(batch_size); + std::vector res_p4(batch_size); + for (size_t batch_i = 0; batch_i < batch_size; ++batch_i) { + res_p1[batch_i] = from_blob( + reinterpret_cast(this->renderer_vec[batch_i].grad_cam_d), + {3}, + this->device_type, + this->device_index, + torch::kFloat, + this->device_type == c10::DeviceType::CUDA +#ifdef WITH_CUDA + ? at::cuda::getCurrentCUDAStream() +#else + ? (cudaStream_t) nullptr +#endif + : (cudaStream_t) nullptr); + res_p2[batch_i] = from_blob( + reinterpret_cast( + this->renderer_vec[batch_i].grad_cam_d + 3), + {3}, + this->device_type, + this->device_index, + torch::kFloat, + this->device_type == c10::DeviceType::CUDA +#ifdef WITH_CUDA + ? at::cuda::getCurrentCUDAStream() +#else + ? (cudaStream_t) nullptr +#endif + : (cudaStream_t) nullptr); + res_p3[batch_i] = from_blob( + reinterpret_cast( + this->renderer_vec[batch_i].grad_cam_d + 6), + {3}, + this->device_type, + this->device_index, + torch::kFloat, + this->device_type == c10::DeviceType::CUDA +#ifdef WITH_CUDA + ? at::cuda::getCurrentCUDAStream() +#else + ? (cudaStream_t) nullptr +#endif + : (cudaStream_t) nullptr); + res_p4[batch_i] = from_blob( + reinterpret_cast( + this->renderer_vec[batch_i].grad_cam_d + 9), + {3}, + this->device_type, + this->device_index, + torch::kFloat, + this->device_type == c10::DeviceType::CUDA +#ifdef WITH_CUDA + ? at::cuda::getCurrentCUDAStream() +#else + ? (cudaStream_t) nullptr +#endif + : (cudaStream_t) nullptr); + } + std::get<3>(ret) = torch::stack(res_p1); + std::get<4>(ret) = torch::stack(res_p2); + std::get<5>(ret) = torch::stack(res_p3); + std::get<6>(ret) = torch::stack(res_p4); + } else { + std::get<3>(ret) = from_blob( + reinterpret_cast(this->renderer_vec[0].grad_cam_d), + {3}, + this->device_type, + this->device_index, + torch::kFloat, + this->device_type == c10::DeviceType::CUDA +#ifdef WITH_CUDA + ? at::cuda::getCurrentCUDAStream() +#else + ? (cudaStream_t) nullptr +#endif + : (cudaStream_t) nullptr); + std::get<4>(ret) = from_blob( + reinterpret_cast(this->renderer_vec[0].grad_cam_d + 3), + {3}, + this->device_type, + this->device_index, + torch::kFloat, + this->device_type == c10::DeviceType::CUDA +#ifdef WITH_CUDA + ? at::cuda::getCurrentCUDAStream() +#else + ? (cudaStream_t) nullptr +#endif + : (cudaStream_t) nullptr); + std::get<5>(ret) = from_blob( + reinterpret_cast(this->renderer_vec[0].grad_cam_d + 6), + {3}, + this->device_type, + this->device_index, + torch::kFloat, + this->device_type == c10::DeviceType::CUDA +#ifdef WITH_CUDA + ? at::cuda::getCurrentCUDAStream() +#else + ? (cudaStream_t) nullptr +#endif + : (cudaStream_t) nullptr); + std::get<6>(ret) = from_blob( + reinterpret_cast(this->renderer_vec[0].grad_cam_d + 9), + {3}, + this->device_type, + this->device_index, + torch::kFloat, + this->device_type == c10::DeviceType::CUDA +#ifdef WITH_CUDA + ? at::cuda::getCurrentCUDAStream() +#else + ? (cudaStream_t) nullptr +#endif + : (cudaStream_t) nullptr); + } + } + if (dif_opy) { + if (batch_processing) { + std::vector results(batch_size); + for (size_t batch_i = 0; batch_i < batch_size; ++batch_i) { + results[batch_i] = from_blob( + reinterpret_cast(this->renderer_vec[batch_i].grad_opy_d), + {static_cast(n_points)}, + this->device_type, + this->device_index, + torch::kFloat, + this->device_type == c10::DeviceType::CUDA +#ifdef WITH_CUDA + ? at::cuda::getCurrentCUDAStream() +#else + ? (cudaStream_t) nullptr +#endif + : (cudaStream_t) nullptr); + } + std::get<7>(ret) = torch::stack(results); + } else { + std::get<7>(ret) = from_blob( + reinterpret_cast(this->renderer_vec[0].grad_opy_d), + {static_cast(n_points)}, + this->device_type, + this->device_index, + torch::kFloat, + this->device_type == c10::DeviceType::CUDA +#ifdef WITH_CUDA + ? at::cuda::getCurrentCUDAStream() +#else + ? (cudaStream_t) nullptr +#endif + : (cudaStream_t) nullptr); + } + } + return ret; +}; + +} // namespace pytorch +} // namespace pulsar diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/pytorch/renderer.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/pytorch/renderer.h new file mode 100644 index 0000000000000000000000000000000000000000..2525ca3f3dd9036320401b2a0059a2d5b6b864a4 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/pytorch/renderer.h @@ -0,0 +1,174 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_PYTORCH_RENDERER_H_ +#define PULSAR_NATIVE_PYTORCH_RENDERER_H_ + +#include "../global.h" +#include "../include/renderer.h" + +namespace pulsar { +namespace pytorch { + +struct Renderer { + public: + /** + * Pytorch Pulsar differentiable rendering module. + */ + explicit Renderer( + const unsigned int& width, + const unsigned int& height, + const uint& max_n_balls, + const bool& orthogonal_projection, + const bool& right_handed_system, + const float& background_normalization_depth, + const uint& n_channels, + const uint& n_track); + ~Renderer(); + + std::tuple forward( + const torch::Tensor& vert_pos, + const torch::Tensor& vert_col, + const torch::Tensor& vert_radii, + const torch::Tensor& cam_pos, + const torch::Tensor& pixel_0_0_center, + const torch::Tensor& pixel_vec_x, + const torch::Tensor& pixel_vec_y, + const torch::Tensor& focal_length, + const torch::Tensor& principal_point_offsets, + const float& gamma, + const float& max_depth, + float min_depth, + const c10::optional& bg_col, + const c10::optional& opacity, + const float& percent_allowed_difference, + const uint& max_n_hits, + const uint& mode); + + std::tuple< + at::optional, + at::optional, + at::optional, + at::optional, + at::optional, + at::optional, + at::optional, + at::optional> + backward( + const torch::Tensor& grad_im, + const torch::Tensor& image, + const torch::Tensor& forw_info, + const torch::Tensor& vert_pos, + const torch::Tensor& vert_col, + const torch::Tensor& vert_radii, + const torch::Tensor& cam_pos, + const torch::Tensor& pixel_0_0_center, + const torch::Tensor& pixel_vec_x, + const torch::Tensor& pixel_vec_y, + const torch::Tensor& focal_length, + const torch::Tensor& principal_point_offsets, + const float& gamma, + const float& max_depth, + float min_depth, + const c10::optional& bg_col, + const c10::optional& opacity, + const float& percent_allowed_difference, + const uint& max_n_hits, + const uint& mode, + const bool& dif_pos, + const bool& dif_col, + const bool& dif_rad, + const bool& dif_cam, + const bool& dif_opy, + const at::optional>& dbg_pos); + + // Infrastructure. + /** + * Ensure that the renderer is placed on this device. + * Is nearly a no-op if the device is correct. + */ + void ensure_on_device(torch::Device device, bool non_blocking = false); + + /** + * Ensure that at least n renderers are available. + */ + void ensure_n_renderers_gte(const size_t& batch_size); + + /** + * Check the parameters. + */ + std::tuple arg_check( + const torch::Tensor& vert_pos, + const torch::Tensor& vert_col, + const torch::Tensor& vert_radii, + const torch::Tensor& cam_pos, + const torch::Tensor& pixel_0_0_center, + const torch::Tensor& pixel_vec_x, + const torch::Tensor& pixel_vec_y, + const torch::Tensor& focal_length, + const torch::Tensor& principal_point_offsets, + const float& gamma, + const float& max_depth, + float& min_depth, + const c10::optional& bg_col, + const c10::optional& opacity, + const float& percent_allowed_difference, + const uint& max_n_hits, + const uint& mode); + + bool operator==(const Renderer& rhs) const; + inline friend std::ostream& operator<<( + std::ostream& stream, + const Renderer& self) { + stream << "pulsar::Renderer["; + // Device info. + stream << self.device_type; + if (self.device_index != -1) + stream << ", ID " << self.device_index; + stream << "]"; + return stream; + } + + inline uint width() const { + return this->renderer_vec[0].cam.film_width; + } + inline uint height() const { + return this->renderer_vec[0].cam.film_height; + } + inline int max_num_balls() const { + return this->renderer_vec[0].max_num_balls; + } + inline bool orthogonal() const { + return this->renderer_vec[0].cam.orthogonal_projection; + } + inline bool right_handed() const { + return this->renderer_vec[0].cam.right_handed; + } + inline uint n_track() const { + return static_cast(this->renderer_vec[0].n_track); + } + + /** A tensor that is registered as a buffer with this Module to track its + * device placement. Unfortunately, pytorch doesn't offer tracking Module + * device placement in a better way as of now. + */ + torch::Tensor device_tracker; + + protected: + /** The device type for this renderer. */ + c10::DeviceType device_type; + /** The device index for this renderer. */ + c10::DeviceIndex device_index; + /** Pointer to the underlying pulsar renderers. */ + std::vector renderer_vec; +}; + +} // namespace pytorch +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/pytorch/tensor_util.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/pytorch/tensor_util.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1dd41ed4e5ef40d2c78b3d71fb5dff7cdaa4e6c0 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/pytorch/tensor_util.cpp @@ -0,0 +1,63 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifdef WITH_CUDA +#include +#include +#endif +#include + +#include "./tensor_util.h" + +namespace pulsar { +namespace pytorch { + +torch::Tensor sphere_ids_from_result_info_nograd( + const torch::Tensor& forw_info) { + torch::Tensor result = torch::zeros( + {forw_info.size(0), + forw_info.size(1), + forw_info.size(2), + (forw_info.size(3) - 3) / 2}, + torch::TensorOptions().device(forw_info.device()).dtype(torch::kInt32)); + // Get the relevant slice, contiguous. + torch::Tensor tmp = + forw_info + .slice( + /*dim=*/3, /*start=*/3, /*end=*/forw_info.size(3), /*step=*/2) + .contiguous(); + if (forw_info.device().type() == c10::DeviceType::CUDA) { +#ifdef WITH_CUDA + cudaMemcpyAsync( + result.data_ptr(), + tmp.data_ptr(), + sizeof(uint32_t) * tmp.size(0) * tmp.size(1) * tmp.size(2) * + tmp.size(3), + cudaMemcpyDeviceToDevice, + at::cuda::getCurrentCUDAStream()); +#else + throw std::runtime_error( + "Copy on CUDA device initiated but built " + "without CUDA support."); +#endif + } else { + memcpy( + result.data_ptr(), + tmp.data_ptr(), + sizeof(uint32_t) * tmp.size(0) * tmp.size(1) * tmp.size(2) * + tmp.size(3)); + } + // `tmp` is freed after this, the memory might get reallocated. However, + // only kernels in the same stream should ever be able to write to this + // memory, which are executed only after the memcpy is complete. That's + // why we can just continue. + return result; +} + +} // namespace pytorch +} // namespace pulsar diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/pytorch/tensor_util.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/pytorch/tensor_util.h new file mode 100644 index 0000000000000000000000000000000000000000..9f1d677cbfd4377f27224e05abc66085a06aa60c --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/pytorch/tensor_util.h @@ -0,0 +1,23 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_PYTORCH_TENSOR_UTIL_H_ +#define PULSAR_NATIVE_PYTORCH_TENSOR_UTIL_H_ + +#include + +namespace pulsar { +namespace pytorch { + +torch::Tensor sphere_ids_from_result_info_nograd( + const torch::Tensor& forw_info); + +} +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/pytorch/util.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/pytorch/util.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7d25b6e8504c765b816e7793419e9de63a7719dd --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/pytorch/util.cpp @@ -0,0 +1,33 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifdef WITH_CUDA +#include + +namespace pulsar { +namespace pytorch { + +void cudaDevToDev( + void* trg, + const void* src, + const int& size, + const cudaStream_t& stream) { + cudaMemcpyAsync(trg, src, size, cudaMemcpyDeviceToDevice, stream); +} + +void cudaDevToHost( + void* trg, + const void* src, + const int& size, + const cudaStream_t& stream) { + cudaMemcpyAsync(trg, src, size, cudaMemcpyDeviceToHost, stream); +} + +} // namespace pytorch +} // namespace pulsar +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/pytorch/util.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/pytorch/util.h new file mode 100644 index 0000000000000000000000000000000000000000..be3dc80defbb78c6e65722a1dda5d70e288e73c7 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/pytorch/util.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef PULSAR_NATIVE_PYTORCH_UTIL_H_ +#define PULSAR_NATIVE_PYTORCH_UTIL_H_ + +#include +#include "../global.h" + +namespace pulsar { +namespace pytorch { + +void cudaDevToDev( + void* trg, + const void* src, + const int& size, + const cudaStream_t& stream); +void cudaDevToHost( + void* trg, + const void* src, + const int& size, + const cudaStream_t& stream); + +/** + * This method takes a memory pointer and wraps it into a pytorch tensor. + * + * This is preferred over `torch::from_blob`, since that requires a CUDA + * managed pointer. However, working with these for high performance + * operations is slower. Most of the rendering operations should stay + * local to the respective GPU anyways, so unmanaged pointers are + * preferred. + */ +template +torch::Tensor from_blob( + const T* ptr, + const torch::IntArrayRef& shape, + const c10::DeviceType& device_type, + const c10::DeviceIndex& device_index, + const torch::Dtype& dtype, + const cudaStream_t& stream) { + torch::Tensor ret = torch::zeros( + shape, torch::device({device_type, device_index}).dtype(dtype)); + const int num_elements = + std::accumulate(shape.begin(), shape.end(), 1, std::multiplies{}); + if (device_type == c10::DeviceType::CUDA) { +#ifdef WITH_CUDA + cudaDevToDev( + ret.data_ptr(), + static_cast(ptr), + sizeof(T) * num_elements, + stream); +#else + throw std::runtime_error( + "Initiating devToDev copy on a build without CUDA."); +#endif + // TODO: check for synchronization. + } else { + memcpy(ret.data_ptr(), ptr, sizeof(T) * num_elements); + } + return ret; +}; + +} // namespace pytorch +} // namespace pulsar + +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/warnings.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/warnings.cpp new file mode 100644 index 0000000000000000000000000000000000000000..54615ac1392db7788c643e93a40b4824b59ec102 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/pulsar/warnings.cpp @@ -0,0 +1,21 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "./global.h" +#include "./logging.h" + +/** + * A compilation unit to provide warnings about the code and avoid + * repeated messages. + */ +#ifdef PULSAR_ASSERTIONS +#pragma message("WARNING: assertions are enabled in Pulsar.") +#endif +#ifdef PULSAR_LOGGING_ENABLED +#pragma message("WARNING: logging is enabled in Pulsar.") +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_coarse/bitmask.cuh b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_coarse/bitmask.cuh new file mode 100644 index 0000000000000000000000000000000000000000..6ffcac87caa13f37a5ccb12b565d33450bc035c2 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_coarse/bitmask.cuh @@ -0,0 +1,79 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once +#define BINMASK_H + +// A BitMask represents a bool array of shape (H, W, N). We pack values into +// the bits of unsigned ints; a single unsigned int has B = 32 bits, so to hold +// all values we use H * W * (N / B) = H * W * D values. We want to store +// BitMasks in shared memory, so we assume that the memory has already been +// allocated for it elsewhere. +class BitMask { + public: + __device__ BitMask(unsigned int* data, int H, int W, int N) + : data(data), H(H), W(W), B(8 * sizeof(unsigned int)), D(N / B) { + // TODO: check if the data is null. + N = ceilf(N % 32); // take ceil incase N % 32 != 0 + block_clear(); // clear the data + } + + // Use all threads in the current block to clear all bits of this BitMask + __device__ void block_clear() { + for (int i = threadIdx.x; i < H * W * D; i += blockDim.x) { + data[i] = 0; + } + __syncthreads(); + } + + __device__ int _get_elem_idx(int y, int x, int d) { + return y * W * D + x * D + d / B; + } + + __device__ int _get_bit_idx(int d) { + return d % B; + } + + // Turn on a single bit (y, x, d) + __device__ void set(int y, int x, int d) { + int elem_idx = _get_elem_idx(y, x, d); + int bit_idx = _get_bit_idx(d); + const unsigned int mask = 1U << bit_idx; + atomicOr(data + elem_idx, mask); + } + + // Turn off a single bit (y, x, d) + __device__ void unset(int y, int x, int d) { + int elem_idx = _get_elem_idx(y, x, d); + int bit_idx = _get_bit_idx(d); + const unsigned int mask = ~(1U << bit_idx); + atomicAnd(data + elem_idx, mask); + } + + // Check whether the bit (y, x, d) is on or off + __device__ bool get(int y, int x, int d) { + int elem_idx = _get_elem_idx(y, x, d); + int bit_idx = _get_bit_idx(d); + return (data[elem_idx] >> bit_idx) & 1U; + } + + // Compute the number of bits set in the row (y, x, :) + __device__ int count(int y, int x) { + int total = 0; + for (int i = 0; i < D; ++i) { + int elem_idx = y * W * D + x * D + i; + unsigned int elem = data[elem_idx]; + total += __popc(elem); + } + return total; + } + + private: + unsigned int* data; + int H, W, B, D; +}; diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_coarse/rasterize_coarse.cu b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_coarse/rasterize_coarse.cu new file mode 100644 index 0000000000000000000000000000000000000000..aed57d21bf1e9c70a7bde543223960894a49ada2 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_coarse/rasterize_coarse.cu @@ -0,0 +1,388 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include +#include +#include +#include +#include "rasterize_coarse/bitmask.cuh" +#include "rasterize_points/rasterization_utils.cuh" +#include "utils/float_math.cuh" +#include "utils/geometry_utils.cuh" // For kEpsilon -- gross + +__global__ void TriangleBoundingBoxKernel( + const float* face_verts, // (F, 3, 3) + const int F, + const float blur_radius, + float* bboxes, // (4, F) + bool* skip_face) { // (F,) + const int tid = blockIdx.x * blockDim.x + threadIdx.x; + const int num_threads = blockDim.x * gridDim.x; + const float sqrt_radius = sqrt(blur_radius); + for (int f = tid; f < F; f += num_threads) { + const float v0x = face_verts[f * 9 + 0 * 3 + 0]; + const float v0y = face_verts[f * 9 + 0 * 3 + 1]; + const float v0z = face_verts[f * 9 + 0 * 3 + 2]; + const float v1x = face_verts[f * 9 + 1 * 3 + 0]; + const float v1y = face_verts[f * 9 + 1 * 3 + 1]; + const float v1z = face_verts[f * 9 + 1 * 3 + 2]; + const float v2x = face_verts[f * 9 + 2 * 3 + 0]; + const float v2y = face_verts[f * 9 + 2 * 3 + 1]; + const float v2z = face_verts[f * 9 + 2 * 3 + 2]; + const float xmin = FloatMin3(v0x, v1x, v2x) - sqrt_radius; + const float xmax = FloatMax3(v0x, v1x, v2x) + sqrt_radius; + const float ymin = FloatMin3(v0y, v1y, v2y) - sqrt_radius; + const float ymax = FloatMax3(v0y, v1y, v2y) + sqrt_radius; + const float zmin = FloatMin3(v0z, v1z, v2z); + const bool skip = zmin < kEpsilon; + bboxes[0 * F + f] = xmin; + bboxes[1 * F + f] = xmax; + bboxes[2 * F + f] = ymin; + bboxes[3 * F + f] = ymax; + skip_face[f] = skip; + } +} + +__global__ void PointBoundingBoxKernel( + const float* points, // (P, 3) + const float* radius, // (P,) + const int P, + float* bboxes, // (4, P) + bool* skip_points) { + const int tid = blockIdx.x * blockDim.x + threadIdx.x; + const int num_threads = blockDim.x * gridDim.x; + for (int p = tid; p < P; p += num_threads) { + const float x = points[p * 3 + 0]; + const float y = points[p * 3 + 1]; + const float z = points[p * 3 + 2]; + const float r = radius[p]; + // TODO: change to kEpsilon to match triangles? + const bool skip = z < 0; + bboxes[0 * P + p] = x - r; + bboxes[1 * P + p] = x + r; + bboxes[2 * P + p] = y - r; + bboxes[3 * P + p] = y + r; + skip_points[p] = skip; + } +} + +__global__ void RasterizeCoarseCudaKernel( + const float* bboxes, // (4, E) (xmin, xmax, ymin, ymax) + const bool* should_skip, // (E,) + const int64_t* elem_first_idxs, + const int64_t* elems_per_batch, + const int N, + const int E, + const int H, + const int W, + const int bin_size, + const int chunk_size, + const int max_elem_per_bin, + int* elems_per_bin, + int* bin_elems) { + extern __shared__ char sbuf[]; + const int M = max_elem_per_bin; + // Integer divide round up + const int num_bins_x = 1 + (W - 1) / bin_size; + const int num_bins_y = 1 + (H - 1) / bin_size; + + // NDC range depends on the ratio of W/H + // The shorter side from (H, W) is given an NDC range of 2.0 and + // the other side is scaled by the ratio of H:W. + const float NDC_x_half_range = NonSquareNdcRange(W, H) / 2.0f; + const float NDC_y_half_range = NonSquareNdcRange(H, W) / 2.0f; + + // Size of half a pixel in NDC units is the NDC half range + // divided by the corresponding image dimension + const float half_pix_x = NDC_x_half_range / W; + const float half_pix_y = NDC_y_half_range / H; + + // This is a boolean array of shape (num_bins_y, num_bins_x, chunk_size) + // stored in shared memory that will track whether each elem in the chunk + // falls into each bin of the image. + BitMask binmask((unsigned int*)sbuf, num_bins_y, num_bins_x, chunk_size); + + // Have each block handle a chunk of elements + const int chunks_per_batch = 1 + (E - 1) / chunk_size; + const int num_chunks = N * chunks_per_batch; + + for (int chunk = blockIdx.x; chunk < num_chunks; chunk += gridDim.x) { + const int batch_idx = chunk / chunks_per_batch; // batch index + const int chunk_idx = chunk % chunks_per_batch; + const int elem_chunk_start_idx = chunk_idx * chunk_size; + + binmask.block_clear(); + const int64_t elem_start_idx = elem_first_idxs[batch_idx]; + const int64_t elem_stop_idx = elem_start_idx + elems_per_batch[batch_idx]; + + // Have each thread handle a different face within the chunk + for (int e = threadIdx.x; e < chunk_size; e += blockDim.x) { + const int e_idx = elem_chunk_start_idx + e; + + // Check that we are still within the same element of the batch + if (e_idx >= elem_stop_idx || e_idx < elem_start_idx) { + continue; + } + + if (should_skip[e_idx]) { + continue; + } + const float xmin = bboxes[0 * E + e_idx]; + const float xmax = bboxes[1 * E + e_idx]; + const float ymin = bboxes[2 * E + e_idx]; + const float ymax = bboxes[3 * E + e_idx]; + + // Brute-force search over all bins; TODO(T54294966) something smarter. + for (int by = 0; by < num_bins_y; ++by) { + // Y coordinate of the top and bottom of the bin. + // PixToNdc gives the location of the center of each pixel, so we + // need to add/subtract a half pixel to get the true extent of the bin. + // Reverse ordering of Y axis so that +Y is upwards in the image. + const float bin_y_min = + PixToNonSquareNdc(by * bin_size, H, W) - half_pix_y; + const float bin_y_max = + PixToNonSquareNdc((by + 1) * bin_size - 1, H, W) + half_pix_y; + const bool y_overlap = (ymin <= bin_y_max) && (bin_y_min < ymax); + + for (int bx = 0; bx < num_bins_x; ++bx) { + // X coordinate of the left and right of the bin. + // Reverse ordering of x axis so that +X is left. + const float bin_x_max = + PixToNonSquareNdc((bx + 1) * bin_size - 1, W, H) + half_pix_x; + const float bin_x_min = + PixToNonSquareNdc(bx * bin_size, W, H) - half_pix_x; + + const bool x_overlap = (xmin <= bin_x_max) && (bin_x_min < xmax); + if (y_overlap && x_overlap) { + binmask.set(by, bx, e); + } + } + } + } + __syncthreads(); + // Now we have processed every elem in the current chunk. We need to + // count the number of elems in each bin so we can write the indices + // out to global memory. We have each thread handle a different bin. + for (int byx = threadIdx.x; byx < num_bins_y * num_bins_x; + byx += blockDim.x) { + const int by = byx / num_bins_x; + const int bx = byx % num_bins_x; + const int count = binmask.count(by, bx); + const int elems_per_bin_idx = + batch_idx * num_bins_y * num_bins_x + by * num_bins_x + bx; + + // This atomically increments the (global) number of elems found + // in the current bin, and gets the previous value of the counter; + // this effectively allocates space in the bin_faces array for the + // elems in the current chunk that fall into this bin. + const int start = atomicAdd(elems_per_bin + elems_per_bin_idx, count); + if (start + count > M) { + // The number of elems in this bin is so big that they won't fit. + // We print a warning using CUDA's printf. This may be invisible + // to notebook users, but apparent to others. It would be nice to + // also have a Python-friendly warning, but it is not obvious + // how to do this without slowing down the normal case. + const char* warning = + "Bin size was too small in the coarse rasterization phase. " + "This caused an overflow, meaning output may be incomplete. " + "To solve, " + "try increasing max_faces_per_bin / max_points_per_bin, " + "decreasing bin_size, " + "or setting bin_size to 0 to use the naive rasterization."; + printf(warning); + continue; + } + + // Now loop over the binmask and write the active bits for this bin + // out to bin_faces. + int next_idx = batch_idx * num_bins_y * num_bins_x * M + + by * num_bins_x * M + bx * M + start; + for (int e = 0; e < chunk_size; ++e) { + if (binmask.get(by, bx, e)) { + // TODO(T54296346) find the correct method for handling errors in + // CUDA. Throw an error if num_faces_per_bin > max_faces_per_bin. + // Either decrease bin size or increase max_faces_per_bin + bin_elems[next_idx] = elem_chunk_start_idx + e; + next_idx++; + } + } + } + __syncthreads(); + } +} + +at::Tensor RasterizeCoarseCuda( + const at::Tensor& bboxes, + const at::Tensor& should_skip, + const at::Tensor& elem_first_idxs, + const at::Tensor& elems_per_batch, + const std::tuple image_size, + const int bin_size, + const int max_elems_per_bin) { + // Set the device for the kernel launch based on the device of the input + at::cuda::CUDAGuard device_guard(bboxes.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + const int H = std::get<0>(image_size); + const int W = std::get<1>(image_size); + + const int E = bboxes.size(1); + const int N = elems_per_batch.size(0); + const int M = max_elems_per_bin; + + // Integer divide round up + const int num_bins_y = 1 + (H - 1) / bin_size; + const int num_bins_x = 1 + (W - 1) / bin_size; + + if (num_bins_y >= kMaxItemsPerBin || num_bins_x >= kMaxItemsPerBin) { + std::stringstream ss; + ss << "In RasterizeCoarseCuda got num_bins_y: " << num_bins_y + << ", num_bins_x: " << num_bins_x << ", " << "; that's too many!"; + AT_ERROR(ss.str()); + } + auto opts = elems_per_batch.options().dtype(at::kInt); + at::Tensor elems_per_bin = at::zeros({N, num_bins_y, num_bins_x}, opts); + at::Tensor bin_elems = at::full({N, num_bins_y, num_bins_x, M}, -1, opts); + + if (bin_elems.numel() == 0) { + AT_CUDA_CHECK(cudaGetLastError()); + return bin_elems; + } + + const int chunk_size = 512; + const size_t shared_size = num_bins_y * num_bins_x * chunk_size / 8; + const size_t blocks = 64; + const size_t threads = 512; + + RasterizeCoarseCudaKernel<<>>( + bboxes.contiguous().data_ptr(), + should_skip.contiguous().data_ptr(), + elem_first_idxs.contiguous().data_ptr(), + elems_per_batch.contiguous().data_ptr(), + N, + E, + H, + W, + bin_size, + chunk_size, + M, + elems_per_bin.data_ptr(), + bin_elems.data_ptr()); + + AT_CUDA_CHECK(cudaGetLastError()); + return bin_elems; +} + +at::Tensor RasterizeMeshesCoarseCuda( + const at::Tensor& face_verts, + const at::Tensor& mesh_to_face_first_idx, + const at::Tensor& num_faces_per_mesh, + const std::tuple image_size, + const float blur_radius, + const int bin_size, + const int max_faces_per_bin) { + TORCH_CHECK( + face_verts.ndimension() == 3 && face_verts.size(1) == 3 && + face_verts.size(2) == 3, + "face_verts must have dimensions (num_faces, 3, 3)"); + + // Check inputs are on the same device + at::TensorArg face_verts_t{face_verts, "face_verts", 1}, + mesh_to_face_first_idx_t{ + mesh_to_face_first_idx, "mesh_to_face_first_idx", 2}, + num_faces_per_mesh_t{num_faces_per_mesh, "num_faces_per_mesh", 3}; + at::CheckedFrom c = "RasterizeMeshesCoarseCuda"; + at::checkAllSameGPU( + c, {face_verts_t, mesh_to_face_first_idx_t, num_faces_per_mesh_t}); + + // Set the device for the kernel launch based on the device of the input + at::cuda::CUDAGuard device_guard(face_verts.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + // Allocate tensors for bboxes and should_skip + const int F = face_verts.size(0); + auto float_opts = face_verts.options().dtype(at::kFloat); + auto bool_opts = face_verts.options().dtype(at::kBool); + at::Tensor bboxes = at::empty({4, F}, float_opts); + at::Tensor should_skip = at::empty({F}, bool_opts); + + // Launch kernel to compute triangle bboxes + const size_t blocks = 128; + const size_t threads = 256; + TriangleBoundingBoxKernel<<>>( + face_verts.contiguous().data_ptr(), + F, + blur_radius, + bboxes.contiguous().data_ptr(), + should_skip.contiguous().data_ptr()); + AT_CUDA_CHECK(cudaGetLastError()); + + return RasterizeCoarseCuda( + bboxes, + should_skip, + mesh_to_face_first_idx, + num_faces_per_mesh, + image_size, + bin_size, + max_faces_per_bin); +} + +at::Tensor RasterizePointsCoarseCuda( + const at::Tensor& points, // (P, 3) + const at::Tensor& cloud_to_packed_first_idx, // (N,) + const at::Tensor& num_points_per_cloud, // (N,) + const std::tuple image_size, + const at::Tensor& radius, + const int bin_size, + const int max_points_per_bin) { + TORCH_CHECK( + points.ndimension() == 2 && points.size(1) == 3, + "points must have dimensions (num_points, 3)"); + + // Check inputs are on the same device + at::TensorArg points_t{points, "points", 1}, + cloud_to_packed_first_idx_t{ + cloud_to_packed_first_idx, "cloud_to_packed_first_idx", 2}, + num_points_per_cloud_t{num_points_per_cloud, "num_points_per_cloud", 3}; + at::CheckedFrom c = "RasterizePointsCoarseCuda"; + at::checkAllSameGPU( + c, {points_t, cloud_to_packed_first_idx_t, num_points_per_cloud_t}); + + // Set the device for the kernel launch based on the device of the input + at::cuda::CUDAGuard device_guard(points.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + // Allocate tensors for bboxes and should_skip + const int P = points.size(0); + auto float_opts = points.options().dtype(at::kFloat); + auto bool_opts = points.options().dtype(at::kBool); + at::Tensor bboxes = at::empty({4, P}, float_opts); + at::Tensor should_skip = at::empty({P}, bool_opts); + + // Launch kernel to compute point bboxes + const size_t blocks = 128; + const size_t threads = 256; + PointBoundingBoxKernel<<>>( + points.contiguous().data_ptr(), + radius.contiguous().data_ptr(), + P, + bboxes.contiguous().data_ptr(), + should_skip.contiguous().data_ptr()); + AT_CUDA_CHECK(cudaGetLastError()); + + return RasterizeCoarseCuda( + bboxes, + should_skip, + cloud_to_packed_first_idx, + num_points_per_cloud, + image_size, + bin_size, + max_points_per_bin); +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_coarse/rasterize_coarse.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_coarse/rasterize_coarse.h new file mode 100644 index 0000000000000000000000000000000000000000..858407cb66b2a252f1b2b223f2adaa2ce8074543 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_coarse/rasterize_coarse.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include + +// Arguments are the same as RasterizeMeshesCoarse from +// rasterize_meshes/rasterize_meshes.h +#ifdef WITH_CUDA +torch::Tensor RasterizeMeshesCoarseCuda( + const torch::Tensor& face_verts, + const torch::Tensor& mesh_to_face_first_idx, + const torch::Tensor& num_faces_per_mesh, + const std::tuple image_size, + const float blur_radius, + const int bin_size, + const int max_faces_per_bin); +#endif + +// Arguments are the same as RasterizePointsCoarse from +// rasterize_points/rasterize_points.h +#ifdef WITH_CUDA +torch::Tensor RasterizePointsCoarseCuda( + const torch::Tensor& points, + const torch::Tensor& cloud_to_packed_first_idx, + const torch::Tensor& num_points_per_cloud, + const std::tuple image_size, + const torch::Tensor& radius, + const int bin_size, + const int max_points_per_bin); +#endif diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_meshes/rasterize_meshes.cu b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_meshes/rasterize_meshes.cu new file mode 100644 index 0000000000000000000000000000000000000000..21ff7e504d9478eec865c3f12e3740313ce3bf88 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_meshes/rasterize_meshes.cu @@ -0,0 +1,823 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "rasterize_points/rasterization_utils.cuh" +#include "utils/float_math.cuh" +#include "utils/geometry_utils.cuh" + +namespace { +// A structure for holding details about a pixel. +struct Pixel { + float z; + int64_t idx; // idx of face + float dist; // abs distance of pixel to face + float3 bary; +}; + +__device__ bool operator<(const Pixel& a, const Pixel& b) { + return a.z < b.z || (a.z == b.z && a.idx < b.idx); +} + +// Get the xyz coordinates of the three vertices for the face given by the +// index face_idx into face_verts. +__device__ thrust::tuple GetSingleFaceVerts( + const float* face_verts, + int face_idx) { + const float x0 = face_verts[face_idx * 9 + 0]; + const float y0 = face_verts[face_idx * 9 + 1]; + const float z0 = face_verts[face_idx * 9 + 2]; + const float x1 = face_verts[face_idx * 9 + 3]; + const float y1 = face_verts[face_idx * 9 + 4]; + const float z1 = face_verts[face_idx * 9 + 5]; + const float x2 = face_verts[face_idx * 9 + 6]; + const float y2 = face_verts[face_idx * 9 + 7]; + const float z2 = face_verts[face_idx * 9 + 8]; + + const float3 v0xyz = make_float3(x0, y0, z0); + const float3 v1xyz = make_float3(x1, y1, z1); + const float3 v2xyz = make_float3(x2, y2, z2); + + return thrust::make_tuple(v0xyz, v1xyz, v2xyz); +} + +// Get the min/max x/y/z values for the face given by vertices v0, v1, v2. +__device__ thrust::tuple +GetFaceBoundingBox(float3 v0, float3 v1, float3 v2) { + const float xmin = FloatMin3(v0.x, v1.x, v2.x); + const float ymin = FloatMin3(v0.y, v1.y, v2.y); + const float zmin = FloatMin3(v0.z, v1.z, v2.z); + const float xmax = FloatMax3(v0.x, v1.x, v2.x); + const float ymax = FloatMax3(v0.y, v1.y, v2.y); + const float zmax = FloatMax3(v0.z, v1.z, v2.z); + + return thrust::make_tuple( + make_float2(xmin, xmax), + make_float2(ymin, ymax), + make_float2(zmin, zmax)); +} + +// Check if the point (px, py) lies outside the face bounding box face_bbox. +// Return true if the point is outside. +__device__ bool CheckPointOutsideBoundingBox( + float3 v0, + float3 v1, + float3 v2, + float blur_radius, + float2 pxy) { + const auto bbox = GetFaceBoundingBox(v0, v1, v2); + const float2 xlims = thrust::get<0>(bbox); + const float2 ylims = thrust::get<1>(bbox); + const float2 zlims = thrust::get<2>(bbox); + + const float x_min = xlims.x - blur_radius; + const float y_min = ylims.x - blur_radius; + const float x_max = xlims.y + blur_radius; + const float y_max = ylims.y + blur_radius; + + // Faces with at least one vertex behind the camera won't render correctly + // and should be removed or clipped before calling the rasterizer + const bool z_invalid = zlims.x < kEpsilon; + + // Check if the current point is oustside the triangle bounding box. + return ( + pxy.x > x_max || pxy.x < x_min || pxy.y > y_max || pxy.y < y_min || + z_invalid); +} + +// This function checks if a pixel given by xy location pxy lies within the +// face with index face_idx in face_verts. One of the inputs is a list (q) +// which contains Pixel structs with the indices of the faces which intersect +// with this pixel sorted by closest z distance. If the point pxy lies in the +// face, the list (q) is updated and re-orderered in place. In addition +// the auxiliary variables q_size, q_max_z and q_max_idx are also modified. +// This code is shared between RasterizeMeshesNaiveCudaKernel and +// RasterizeMeshesFineCudaKernel. +template +__device__ void CheckPixelInsideFace( + const float* face_verts, // (F, 3, 3) + const int64_t* clipped_faces_neighbor_idx, // (F,) + const int face_idx, + int& q_size, + float& q_max_z, + int& q_max_idx, + FaceQ& q, + const float blur_radius, + const float2 pxy, // Coordinates of the pixel + const int K, + const bool perspective_correct, + const bool clip_barycentric_coords, + const bool cull_backfaces) { + const auto v012 = GetSingleFaceVerts(face_verts, face_idx); + const float3 v0 = thrust::get<0>(v012); + const float3 v1 = thrust::get<1>(v012); + const float3 v2 = thrust::get<2>(v012); + + // Only need xy for barycentric coordinates and distance calculations. + const float2 v0xy = make_float2(v0.x, v0.y); + const float2 v1xy = make_float2(v1.x, v1.y); + const float2 v2xy = make_float2(v2.x, v2.y); + + // Perform checks and skip if: + // 1. the face is behind the camera + // 2. the face is facing away from the camera + // 3. the face has very small face area + // 4. the pixel is outside the face bbox + const float zmax = FloatMax3(v0.z, v1.z, v2.z); + const bool outside_bbox = CheckPointOutsideBoundingBox( + v0, v1, v2, sqrt(blur_radius), pxy); // use sqrt of blur for bbox + const float face_area = EdgeFunctionForward(v0xy, v1xy, v2xy); + // Check if the face is visible to the camera. + const bool back_face = face_area < 0.0; + const bool zero_face_area = + (face_area <= kEpsilon && face_area >= -1.0f * kEpsilon); + + if (zmax < 0 || cull_backfaces && back_face || outside_bbox || + zero_face_area) { + return; + } + + // Calculate barycentric coords and euclidean dist to triangle. + const float3 p_bary0 = BarycentricCoordsForward(pxy, v0xy, v1xy, v2xy); + const float3 p_bary = !perspective_correct + ? p_bary0 + : BarycentricPerspectiveCorrectionForward(p_bary0, v0.z, v1.z, v2.z); + const float3 p_bary_clip = + !clip_barycentric_coords ? p_bary : BarycentricClipForward(p_bary); + + const float pz = + p_bary_clip.x * v0.z + p_bary_clip.y * v1.z + p_bary_clip.z * v2.z; + + if (pz < 0) { + return; // Face is behind the image plane. + } + + // Get abs squared distance + const float dist = PointTriangleDistanceForward(pxy, v0xy, v1xy, v2xy); + + // Use the unclipped bary coordinates to determine if the point is inside the + // face. + const bool inside = p_bary.x > 0.0f && p_bary.y > 0.0f && p_bary.z > 0.0f; + const float signed_dist = inside ? -dist : dist; + // Check if pixel is outside blur region + if (!inside && dist >= blur_radius) { + return; + } + + // Handle the case where a face (f) partially behind the image plane is + // clipped to a quadrilateral and then split into two faces (t1, t2). In this + // case we: + // 1. Find the index of the neighboring face (e.g. for t1 need index of t2) + // 2. Check if the neighboring face (t2) is already in the top K faces + // 3. If yes, compare the distance of the pixel to t1 with the distance to t2. + // 4. If dist_t1 < dist_t2, overwrite the values for t2 in the top K faces. + const int neighbor_idx = clipped_faces_neighbor_idx[face_idx]; + int neighbor_idx_top_k = -1; + + // Check if neighboring face is already in the top K. + // -1 is the fill value in clipped_faces_neighbor_idx + if (neighbor_idx != -1) { + // Only need to loop until q_size. + for (int i = 0; i < q_size; i++) { + if (q[i].idx == neighbor_idx) { + neighbor_idx_top_k = i; + break; + } + } + } + // If neighbor idx is not -1 then it is in the top K struct. + if (neighbor_idx_top_k != -1) { + // If dist of current face is less than neighbor then overwrite the + // neighbor face values in the top K struct. + float neighbor_dist = abs(q[neighbor_idx_top_k].dist); + if (dist < neighbor_dist) { + // Overwrite the neighbor face values + q[neighbor_idx_top_k] = {pz, face_idx, signed_dist, p_bary_clip}; + + // If pz > q_max then overwrite the max values and index of the max. + // q_size stays the same. + if (pz > q_max_z) { + q_max_z = pz; + q_max_idx = neighbor_idx_top_k; + } + } + } else { + // Handle as a normal face + if (q_size < K) { + // Just insert it. + q[q_size] = {pz, face_idx, signed_dist, p_bary_clip}; + if (pz > q_max_z) { + q_max_z = pz; + q_max_idx = q_size; + } + q_size++; + } else if (pz < q_max_z) { + // Overwrite the old max, and find the new max. + q[q_max_idx] = {pz, face_idx, signed_dist, p_bary_clip}; + q_max_z = pz; + for (int i = 0; i < K; i++) { + if (q[i].z > q_max_z) { + q_max_z = q[i].z; + q_max_idx = i; + } + } + } + } +} + +} // namespace + +// **************************************************************************** +// * NAIVE RASTERIZATION * +// **************************************************************************** +__global__ void RasterizeMeshesNaiveCudaKernel( + const float* face_verts, + const int64_t* mesh_to_face_first_idx, + const int64_t* num_faces_per_mesh, + const int64_t* clipped_faces_neighbor_idx, + const float blur_radius, + const bool perspective_correct, + const bool clip_barycentric_coords, + const bool cull_backfaces, + const int N, + const int H, + const int W, + const int K, + int64_t* face_idxs, + float* zbuf, + float* pix_dists, + float* bary) { + // Simple version: One thread per output pixel + int num_threads = gridDim.x * blockDim.x; + int tid = blockDim.x * blockIdx.x + threadIdx.x; + + for (int i = tid; i < N * H * W; i += num_threads) { + // Convert linear index to 3D index + const int n = i / (H * W); // batch index. + const int pix_idx = i % (H * W); + + // Reverse ordering of X and Y axes + const int yi = H - 1 - pix_idx / W; + const int xi = W - 1 - pix_idx % W; + + // screen coordinates to ndc coordinates of pixel. + const float xf = PixToNonSquareNdc(xi, W, H); + const float yf = PixToNonSquareNdc(yi, H, W); + const float2 pxy = make_float2(xf, yf); + + // For keeping track of the K closest points we want a data structure + // that (1) gives O(1) access to the closest point for easy comparisons, + // and (2) allows insertion of new elements. In the CPU version we use + // std::priority_queue; then (2) is O(log K). We can't use STL + // containers in CUDA; we could roll our own max heap in an array, but + // that would likely have a lot of warp divergence so we do something + // simpler instead: keep the elements in an unsorted array, but keep + // track of the max value and the index of the max value. Then (1) is + // still O(1) time, while (2) is O(K) with a clean loop. Since K <= 8 + // this should be fast enough for our purposes. + Pixel q[kMaxPointsPerPixel]; + int q_size = 0; + float q_max_z = -1000; + int q_max_idx = -1; + + // Using the batch index of the thread get the start and stop + // indices for the faces. + const int64_t face_start_idx = mesh_to_face_first_idx[n]; + const int64_t face_stop_idx = face_start_idx + num_faces_per_mesh[n]; + + // Loop through the faces in the mesh. + for (int f = face_start_idx; f < face_stop_idx; ++f) { + // Check if the pixel pxy is inside the face bounding box and if it is, + // update q, q_size, q_max_z and q_max_idx in place. + + CheckPixelInsideFace( + face_verts, + clipped_faces_neighbor_idx, + f, + q_size, + q_max_z, + q_max_idx, + q, + blur_radius, + pxy, + K, + perspective_correct, + clip_barycentric_coords, + cull_backfaces); + } + + // TODO: make sorting an option as only top k is needed, not sorted values. + BubbleSort(q, q_size); + int idx = n * H * W * K + pix_idx * K; + + for (int k = 0; k < q_size; ++k) { + face_idxs[idx + k] = q[k].idx; + zbuf[idx + k] = q[k].z; + pix_dists[idx + k] = q[k].dist; + bary[(idx + k) * 3 + 0] = q[k].bary.x; + bary[(idx + k) * 3 + 1] = q[k].bary.y; + bary[(idx + k) * 3 + 2] = q[k].bary.z; + } + } +} + +std::tuple +RasterizeMeshesNaiveCuda( + const at::Tensor& face_verts, + const at::Tensor& mesh_to_faces_packed_first_idx, + const at::Tensor& num_faces_per_mesh, + const at::Tensor& clipped_faces_neighbor_idx, + const std::tuple image_size, + const float blur_radius, + const int num_closest, + const bool perspective_correct, + const bool clip_barycentric_coords, + const bool cull_backfaces) { + TORCH_CHECK( + face_verts.ndimension() == 3 && face_verts.size(1) == 3 && + face_verts.size(2) == 3, + "face_verts must have dimensions (num_faces, 3, 3)"); + + TORCH_CHECK( + num_faces_per_mesh.size(0) == mesh_to_faces_packed_first_idx.size(0), + "num_faces_per_mesh must have save size first dimension as mesh_to_faces_packed_first_idx"); + + TORCH_CHECK( + clipped_faces_neighbor_idx.size(0) == face_verts.size(0), + "clipped_faces_neighbor_idx must have save size first dimension as face_verts"); + + if (num_closest > kMaxPointsPerPixel) { + std::stringstream ss; + ss << "Must have points_per_pixel <= " << kMaxPointsPerPixel; + AT_ERROR(ss.str()); + } + + // Check inputs are on the same device + at::TensorArg face_verts_t{face_verts, "face_verts", 1}, + mesh_to_faces_packed_first_idx_t{ + mesh_to_faces_packed_first_idx, "mesh_to_faces_packed_first_idx", 2}, + num_faces_per_mesh_t{num_faces_per_mesh, "num_faces_per_mesh", 3}, + clipped_faces_neighbor_idx_t{ + clipped_faces_neighbor_idx, "clipped_faces_neighbor_idx", 4}; + at::CheckedFrom c = "RasterizeMeshesNaiveCuda"; + at::checkAllSameGPU( + c, + {face_verts_t, + mesh_to_faces_packed_first_idx_t, + num_faces_per_mesh_t, + clipped_faces_neighbor_idx_t}); + + // Set the device for the kernel launch based on the device of the input + at::cuda::CUDAGuard device_guard(face_verts.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + const int N = num_faces_per_mesh.size(0); // batch size. + const int H = std::get<0>(image_size); + const int W = std::get<1>(image_size); + const int K = num_closest; + + auto long_opts = num_faces_per_mesh.options().dtype(at::kLong); + auto float_opts = face_verts.options().dtype(at::kFloat); + + at::Tensor face_idxs = at::full({N, H, W, K}, -1, long_opts); + at::Tensor zbuf = at::full({N, H, W, K}, -1, float_opts); + at::Tensor pix_dists = at::full({N, H, W, K}, -1, float_opts); + at::Tensor bary = at::full({N, H, W, K, 3}, -1, float_opts); + + if (face_idxs.numel() == 0) { + AT_CUDA_CHECK(cudaGetLastError()); + return std::make_tuple(face_idxs, zbuf, bary, pix_dists); + } + + const size_t blocks = 1024; + const size_t threads = 64; + + RasterizeMeshesNaiveCudaKernel<<>>( + face_verts.contiguous().data_ptr(), + mesh_to_faces_packed_first_idx.contiguous().data_ptr(), + num_faces_per_mesh.contiguous().data_ptr(), + clipped_faces_neighbor_idx.contiguous().data_ptr(), + blur_radius, + perspective_correct, + clip_barycentric_coords, + cull_backfaces, + N, + H, + W, + K, + face_idxs.data_ptr(), + zbuf.data_ptr(), + pix_dists.data_ptr(), + bary.data_ptr()); + + AT_CUDA_CHECK(cudaGetLastError()); + return std::make_tuple(face_idxs, zbuf, bary, pix_dists); +} + +// **************************************************************************** +// * BACKWARD PASS * +// **************************************************************************** +// TODO: benchmark parallelizing over faces_verts instead of over pixels. +__global__ void RasterizeMeshesBackwardCudaKernel( + const float* face_verts, // (F, 3, 3) + const int64_t* pix_to_face, // (N, H, W, K) + const bool perspective_correct, + const bool clip_barycentric_coords, + const int N, + const int H, + const int W, + const int K, + const float* grad_zbuf, // (N, H, W, K) + const float* grad_bary, // (N, H, W, K, 3) + const float* grad_dists, // (N, H, W, K) + float* grad_face_verts) { // (F, 3, 3) + + // Parallelize over each pixel in images of + // size H * W, for each image in the batch of size N. + const int num_threads = gridDim.x * blockDim.x; + const int tid = blockIdx.x * blockDim.x + threadIdx.x; + + for (int t_i = tid; t_i < N * H * W; t_i += num_threads) { + // Convert linear index to 3D index + const int n = t_i / (H * W); // batch index. + const int pix_idx = t_i % (H * W); + + // Reverse ordering of X and Y axes. + const int yi = H - 1 - pix_idx / W; + const int xi = W - 1 - pix_idx % W; + + const float xf = PixToNonSquareNdc(xi, W, H); + const float yf = PixToNonSquareNdc(yi, H, W); + const float2 pxy = make_float2(xf, yf); + + // Loop over all the faces for this pixel. + for (int k = 0; k < K; k++) { + // Index into (N, H, W, K, :) grad tensors + // pixel index + top k index + int i = n * H * W * K + pix_idx * K + k; + + const int f = pix_to_face[i]; + if (f < 0) { + continue; // padded face. + } + // Get xyz coordinates of the three face vertices. + const auto v012 = GetSingleFaceVerts(face_verts, f); + const float3 v0 = thrust::get<0>(v012); + const float3 v1 = thrust::get<1>(v012); + const float3 v2 = thrust::get<2>(v012); + + // Only neex xy for barycentric coordinate and distance calculations. + const float2 v0xy = make_float2(v0.x, v0.y); + const float2 v1xy = make_float2(v1.x, v1.y); + const float2 v2xy = make_float2(v2.x, v2.y); + + // Get upstream gradients for the face. + const float grad_dist_upstream = grad_dists[i]; + const float grad_zbuf_upstream = grad_zbuf[i]; + const float grad_bary_upstream_w0 = grad_bary[i * 3 + 0]; + const float grad_bary_upstream_w1 = grad_bary[i * 3 + 1]; + const float grad_bary_upstream_w2 = grad_bary[i * 3 + 2]; + const float3 grad_bary_upstream = make_float3( + grad_bary_upstream_w0, grad_bary_upstream_w1, grad_bary_upstream_w2); + + const float3 b_w = BarycentricCoordsForward(pxy, v0xy, v1xy, v2xy); + const float3 b_pp = !perspective_correct + ? b_w + : BarycentricPerspectiveCorrectionForward(b_w, v0.z, v1.z, v2.z); + + const float3 b_w_clip = + !clip_barycentric_coords ? b_pp : BarycentricClipForward(b_pp); + + const bool inside = b_pp.x > 0.0f && b_pp.y > 0.0f && b_pp.z > 0.0f; + const float sign = inside ? -1.0f : 1.0f; + + auto grad_dist_f = PointTriangleDistanceBackward( + pxy, v0xy, v1xy, v2xy, sign * grad_dist_upstream); + const float2 ddist_d_v0 = thrust::get<1>(grad_dist_f); + const float2 ddist_d_v1 = thrust::get<2>(grad_dist_f); + const float2 ddist_d_v2 = thrust::get<3>(grad_dist_f); + + // Upstream gradient for barycentric coords from zbuf calculation: + // zbuf = bary_w0 * z0 + bary_w1 * z1 + bary_w2 * z2 + // Therefore + // d_zbuf/d_bary_w0 = z0 + // d_zbuf/d_bary_w1 = z1 + // d_zbuf/d_bary_w2 = z2 + const float3 d_zbuf_d_bwclip = make_float3(v0.z, v1.z, v2.z); + + // Total upstream barycentric gradients are the sum of + // external upstream gradients and contribution from zbuf. + const float3 grad_bary_f_sum = + (grad_bary_upstream + grad_zbuf_upstream * d_zbuf_d_bwclip); + + float3 grad_bary0 = grad_bary_f_sum; + + if (clip_barycentric_coords) { + grad_bary0 = BarycentricClipBackward(b_w, grad_bary_f_sum); + } + + float dz0_persp = 0.0f, dz1_persp = 0.0f, dz2_persp = 0.0f; + if (perspective_correct) { + auto perspective_grads = BarycentricPerspectiveCorrectionBackward( + b_w, v0.z, v1.z, v2.z, grad_bary0); + grad_bary0 = thrust::get<0>(perspective_grads); + dz0_persp = thrust::get<1>(perspective_grads); + dz1_persp = thrust::get<2>(perspective_grads); + dz2_persp = thrust::get<3>(perspective_grads); + } + + auto grad_bary_f = + BarycentricCoordsBackward(pxy, v0xy, v1xy, v2xy, grad_bary0); + const float2 dbary_d_v0 = thrust::get<1>(grad_bary_f); + const float2 dbary_d_v1 = thrust::get<2>(grad_bary_f); + const float2 dbary_d_v2 = thrust::get<3>(grad_bary_f); + + atomicAdd(grad_face_verts + f * 9 + 0, dbary_d_v0.x + ddist_d_v0.x); + atomicAdd(grad_face_verts + f * 9 + 1, dbary_d_v0.y + ddist_d_v0.y); + atomicAdd( + grad_face_verts + f * 9 + 2, + grad_zbuf_upstream * b_w_clip.x + dz0_persp); + atomicAdd(grad_face_verts + f * 9 + 3, dbary_d_v1.x + ddist_d_v1.x); + atomicAdd(grad_face_verts + f * 9 + 4, dbary_d_v1.y + ddist_d_v1.y); + atomicAdd( + grad_face_verts + f * 9 + 5, + grad_zbuf_upstream * b_w_clip.y + dz1_persp); + atomicAdd(grad_face_verts + f * 9 + 6, dbary_d_v2.x + ddist_d_v2.x); + atomicAdd(grad_face_verts + f * 9 + 7, dbary_d_v2.y + ddist_d_v2.y); + atomicAdd( + grad_face_verts + f * 9 + 8, + grad_zbuf_upstream * b_w_clip.z + dz2_persp); + } + } +} + +at::Tensor RasterizeMeshesBackwardCuda( + const at::Tensor& face_verts, // (F, 3, 3) + const at::Tensor& pix_to_face, // (N, H, W, K) + const at::Tensor& grad_zbuf, // (N, H, W, K) + const at::Tensor& grad_bary, // (N, H, W, K, 3) + const at::Tensor& grad_dists, // (N, H, W, K) + const bool perspective_correct, + const bool clip_barycentric_coords) { + // Check inputs are on the same device + at::TensorArg face_verts_t{face_verts, "face_verts", 1}, + pix_to_face_t{pix_to_face, "pix_to_face", 2}, + grad_zbuf_t{grad_zbuf, "grad_zbuf", 3}, + grad_bary_t{grad_bary, "grad_bary", 4}, + grad_dists_t{grad_dists, "grad_dists", 5}; + at::CheckedFrom c = "RasterizeMeshesBackwardCuda"; + at::checkAllSameGPU( + c, {face_verts_t, pix_to_face_t, grad_zbuf_t, grad_bary_t, grad_dists_t}); + at::checkAllSameType( + c, {face_verts_t, grad_zbuf_t, grad_bary_t, grad_dists_t}); + + // This is nondeterministic because atomicAdd + at::globalContext().alertNotDeterministic("RasterizeMeshesBackwardCuda"); + + // Set the device for the kernel launch based on the device of the input + at::cuda::CUDAGuard device_guard(face_verts.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + const int F = face_verts.size(0); + const int N = pix_to_face.size(0); + const int H = pix_to_face.size(1); + const int W = pix_to_face.size(2); + const int K = pix_to_face.size(3); + + at::Tensor grad_face_verts = at::zeros({F, 3, 3}, face_verts.options()); + + if (grad_face_verts.numel() == 0) { + AT_CUDA_CHECK(cudaGetLastError()); + return grad_face_verts; + } + + const size_t blocks = 1024; + const size_t threads = 64; + + RasterizeMeshesBackwardCudaKernel<<>>( + face_verts.contiguous().data_ptr(), + pix_to_face.contiguous().data_ptr(), + perspective_correct, + clip_barycentric_coords, + N, + H, + W, + K, + grad_zbuf.contiguous().data_ptr(), + grad_bary.contiguous().data_ptr(), + grad_dists.contiguous().data_ptr(), + grad_face_verts.data_ptr()); + + AT_CUDA_CHECK(cudaGetLastError()); + return grad_face_verts; +} + +// **************************************************************************** +// * FINE RASTERIZATION * +// **************************************************************************** +__global__ void RasterizeMeshesFineCudaKernel( + const float* face_verts, // (F, 3, 3) + const int32_t* bin_faces, // (N, BH, BW, T) + const int64_t* clipped_faces_neighbor_idx, // (F,) + const float blur_radius, + const int bin_size, + const bool perspective_correct, + const bool clip_barycentric_coords, + const bool cull_backfaces, + const int N, + const int BH, + const int BW, + const int M, + const int H, + const int W, + const int K, + int64_t* face_idxs, // (N, H, W, K) + float* zbuf, // (N, H, W, K) + float* pix_dists, // (N, H, W, K) + float* bary // (N, H, W, K, 3) +) { + // This can be more than H * W if H or W are not divisible by bin_size. + int num_pixels = N * BH * BW * bin_size * bin_size; + int num_threads = gridDim.x * blockDim.x; + int tid = blockIdx.x * blockDim.x + threadIdx.x; + + for (int pid = tid; pid < num_pixels; pid += num_threads) { + // Convert linear index into bin and pixel indices. We make the within + // block pixel ids move the fastest, so that adjacent threads will fall + // into the same bin; this should give them coalesced memory reads when + // they read from faces and bin_faces. + int i = pid; + const int n = i / (BH * BW * bin_size * bin_size); + i %= BH * BW * bin_size * bin_size; + // bin index y + const int by = i / (BW * bin_size * bin_size); + i %= BW * bin_size * bin_size; + // bin index y + const int bx = i / (bin_size * bin_size); + // pixel within the bin + i %= bin_size * bin_size; + + // Pixel x, y indices + const int yi = i / bin_size + by * bin_size; + const int xi = i % bin_size + bx * bin_size; + + if (yi >= H || xi >= W) + continue; + + const float xf = PixToNonSquareNdc(xi, W, H); + const float yf = PixToNonSquareNdc(yi, H, W); + + const float2 pxy = make_float2(xf, yf); + + // This part looks like the naive rasterization kernel, except we use + // bin_faces to only look at a subset of faces already known to fall + // in this bin. TODO abstract out this logic into some data structure + // that is shared by both kernels? + Pixel q[kMaxPointsPerPixel]; + int q_size = 0; + float q_max_z = -1000; + int q_max_idx = -1; + + for (int m = 0; m < M; m++) { + const int f = bin_faces[n * BH * BW * M + by * BW * M + bx * M + m]; + if (f < 0) { + continue; // bin_faces uses -1 as a sentinal value. + } + // Check if the pixel pxy is inside the face bounding box and if it is, + // update q, q_size, q_max_z and q_max_idx in place. + CheckPixelInsideFace( + face_verts, + clipped_faces_neighbor_idx, + f, + q_size, + q_max_z, + q_max_idx, + q, + blur_radius, + pxy, + K, + perspective_correct, + clip_barycentric_coords, + cull_backfaces); + } + + // Now we've looked at all the faces for this bin, so we can write + // output for the current pixel. + // TODO: make sorting an option as only top k is needed, not sorted values. + BubbleSort(q, q_size); + + // Reverse ordering of the X and Y axis so that + // in the image +Y is pointing up and +X is pointing left. + const int yidx = H - 1 - yi; + const int xidx = W - 1 - xi; + + const int pix_idx = n * H * W * K + yidx * W * K + xidx * K; + for (int k = 0; k < q_size; k++) { + face_idxs[pix_idx + k] = q[k].idx; + zbuf[pix_idx + k] = q[k].z; + pix_dists[pix_idx + k] = q[k].dist; + bary[(pix_idx + k) * 3 + 0] = q[k].bary.x; + bary[(pix_idx + k) * 3 + 1] = q[k].bary.y; + bary[(pix_idx + k) * 3 + 2] = q[k].bary.z; + } + } +} + +std::tuple +RasterizeMeshesFineCuda( + const at::Tensor& face_verts, + const at::Tensor& bin_faces, + const at::Tensor& clipped_faces_neighbor_idx, + const std::tuple image_size, + const float blur_radius, + const int bin_size, + const int faces_per_pixel, + const bool perspective_correct, + const bool clip_barycentric_coords, + const bool cull_backfaces) { + TORCH_CHECK( + face_verts.ndimension() == 3 && face_verts.size(1) == 3 && + face_verts.size(2) == 3, + "face_verts must have dimensions (num_faces, 3, 3)"); + TORCH_CHECK(bin_faces.ndimension() == 4, "bin_faces must have 4 dimensions"); + TORCH_CHECK( + clipped_faces_neighbor_idx.size(0) == face_verts.size(0), + "clipped_faces_neighbor_idx must have the same first dimension as face_verts"); + + // Check inputs are on the same device + at::TensorArg face_verts_t{face_verts, "face_verts", 1}, + bin_faces_t{bin_faces, "bin_faces", 2}, + clipped_faces_neighbor_idx_t{ + clipped_faces_neighbor_idx, "clipped_faces_neighbor_idx", 3}; + at::CheckedFrom c = "RasterizeMeshesFineCuda"; + at::checkAllSameGPU( + c, {face_verts_t, bin_faces_t, clipped_faces_neighbor_idx_t}); + + // Set the device for the kernel launch based on the device of the input + at::cuda::CUDAGuard device_guard(face_verts.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + // bin_faces shape (N, BH, BW, M) + const int N = bin_faces.size(0); + const int BH = bin_faces.size(1); + const int BW = bin_faces.size(2); + const int M = bin_faces.size(3); + const int K = faces_per_pixel; + + const int H = std::get<0>(image_size); + const int W = std::get<1>(image_size); + + if (K > kMaxPointsPerPixel) { + AT_ERROR("Must have num_closest <= 150"); + } + auto long_opts = bin_faces.options().dtype(at::kLong); + auto float_opts = face_verts.options().dtype(at::kFloat); + + at::Tensor face_idxs = at::full({N, H, W, K}, -1, long_opts); + at::Tensor zbuf = at::full({N, H, W, K}, -1, float_opts); + at::Tensor pix_dists = at::full({N, H, W, K}, -1, float_opts); + at::Tensor bary = at::full({N, H, W, K, 3}, -1, float_opts); + + if (face_idxs.numel() == 0) { + AT_CUDA_CHECK(cudaGetLastError()); + return std::make_tuple(face_idxs, zbuf, bary, pix_dists); + } + + const size_t blocks = 1024; + const size_t threads = 64; + + RasterizeMeshesFineCudaKernel<<>>( + face_verts.contiguous().data_ptr(), + bin_faces.contiguous().data_ptr(), + clipped_faces_neighbor_idx.contiguous().data_ptr(), + blur_radius, + bin_size, + perspective_correct, + clip_barycentric_coords, + cull_backfaces, + N, + BH, + BW, + M, + H, + W, + K, + face_idxs.data_ptr(), + zbuf.data_ptr(), + pix_dists.data_ptr(), + bary.data_ptr()); + + return std::make_tuple(face_idxs, zbuf, bary, pix_dists); +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_meshes/rasterize_meshes.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_meshes/rasterize_meshes.h new file mode 100644 index 0000000000000000000000000000000000000000..584aa0238ad86434567716d5c77f212a394b1d84 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_meshes/rasterize_meshes.h @@ -0,0 +1,549 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once +#include +#include +#include +#include "rasterize_coarse/rasterize_coarse.h" +#include "utils/pytorch3d_cutils.h" + +// **************************************************************************** +// * FORWARD PASS * +// **************************************************************************** + +std::tuple +RasterizeMeshesNaiveCpu( + const torch::Tensor& face_verts, + const torch::Tensor& mesh_to_face_first_idx, + const torch::Tensor& num_faces_per_mesh, + const torch::Tensor& clipped_faces_neighbor_idx, + const std::tuple image_size, + const float blur_radius, + const int faces_per_pixel, + const bool perspective_correct, + const bool clip_barycentric_coords, + const bool cull_backfaces); + +#ifdef WITH_CUDA +std::tuple +RasterizeMeshesNaiveCuda( + const at::Tensor& face_verts, + const at::Tensor& mesh_to_face_first_idx, + const at::Tensor& num_faces_per_mesh, + const torch::Tensor& clipped_faces_neighbor_idx, + const std::tuple image_size, + const float blur_radius, + const int num_closest, + const bool perspective_correct, + const bool clip_barycentric_coords, + const bool cull_backfaces); +#endif +// Forward pass for rasterizing a batch of meshes. +// +// Args: +// face_verts: Tensor of shape (F, 3, 3) giving (packed) vertex positions for +// faces in all the meshes in the batch. Concretely, +// face_verts[f, i] = [x, y, z] gives the coordinates for the +// ith vertex of the fth face. These vertices are expected to be +// in NDC coordinates in the range [-1, 1]. +// mesh_to_face_first_idx: LongTensor of shape (N) giving the index in +// faces_verts of the first face in each mesh in +// the batch where N is the batch size. +// num_faces_per_mesh: LongTensor of shape (N) giving the number of faces +// for each mesh in the batch. +// clipped_faces_neighbor_idx: LongTensor of shape (F,) giving the +// index of the neighboring face for each face which was clipped to a +// quadrilateral and then divided into two triangles. +// e.g. for a face f partially behind the image plane which is split into +// two triangles (t1, t2): clipped_faces_neighbor_idx[t1_idx] = t2_idx +// Faces which are not clipped and subdivided are set to -1. +// image_size: Tuple (H, W) giving the size in pixels of the output +// image to be rasterized. +// blur_radius: float distance in NDC coordinates uses to expand the face +// bounding boxes for the rasterization. Set to 0.0 if no blur +// is required. +// faces_per_pixel: the number of closeset faces to rasterize per pixel. +// perspective_correct: Whether to apply perspective correction when +// computing barycentric coordinates. If this is True, +// then this function returns world-space barycentric +// coordinates for each pixel; if this is False then +// this function instead returns screen-space +// barycentric coordinates for each pixel. +// clip_barycentric_coords: Whether, after any perspective correction +// is applied but before the depth is calculated (e.g. for +// z clipping), to "correct" a location outside the face (i.e. with +// a negative barycentric coordinate) to a position on the edge of the +// face. +// cull_backfaces: Bool, Whether to only rasterize mesh faces which are +// visible to the camera. This assumes that vertices of +// front-facing triangles are ordered in an anti-clockwise +// fashion, and triangles that face away from the camera are +// in a clockwise order relative to the current view +// direction. NOTE: This will only work if the mesh faces are +// consistently defined with counter-clockwise ordering when +// viewed from the outside. +// +// Returns: +// A 4 element tuple of: +// pix_to_face: int64 tensor of shape (N, H, W, K) giving the face index of +// each of the closest faces to the pixel in the rasterized +// image, or -1 for pixels that are not covered by any face. +// zbuf: float32 Tensor of shape (N, H, W, K) giving the depth of each of +// the closest faces for each pixel. +// barycentric_coords: float tensor of shape (N, H, W, K, 3) giving +// barycentric coordinates of the pixel with respect to +// each of the closest faces along the z axis, padded +// with -1 for pixels hit by fewer than +// faces_per_pixel faces. +// dists: float tensor of shape (N, H, W, K) giving the euclidean distance +// in the (NDC) x/y plane between each pixel and its K closest +// faces along the z axis padded with -1 for pixels hit by fewer than +// faces_per_pixel faces. +inline std::tuple +RasterizeMeshesNaive( + const torch::Tensor& face_verts, + const torch::Tensor& mesh_to_face_first_idx, + const torch::Tensor& num_faces_per_mesh, + const torch::Tensor& clipped_faces_neighbor_idx, + const std::tuple image_size, + const float blur_radius, + const int faces_per_pixel, + const bool perspective_correct, + const bool clip_barycentric_coords, + const bool cull_backfaces) { + // TODO: Better type checking. + if (face_verts.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(face_verts); + CHECK_CUDA(mesh_to_face_first_idx); + CHECK_CUDA(num_faces_per_mesh); + return RasterizeMeshesNaiveCuda( + face_verts, + mesh_to_face_first_idx, + num_faces_per_mesh, + clipped_faces_neighbor_idx, + image_size, + blur_radius, + faces_per_pixel, + perspective_correct, + clip_barycentric_coords, + cull_backfaces); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } else { + return RasterizeMeshesNaiveCpu( + face_verts, + mesh_to_face_first_idx, + num_faces_per_mesh, + clipped_faces_neighbor_idx, + image_size, + blur_radius, + faces_per_pixel, + perspective_correct, + clip_barycentric_coords, + cull_backfaces); + } +} + +// **************************************************************************** +// * BACKWARD PASS * +// **************************************************************************** + +torch::Tensor RasterizeMeshesBackwardCpu( + const torch::Tensor& face_verts, + const torch::Tensor& pix_to_face, + const torch::Tensor& grad_zbuf, + const torch::Tensor& grad_bary, + const torch::Tensor& grad_dists, + const bool perspective_correct, + const bool clip_barycentric_coords); + +#ifdef WITH_CUDA +torch::Tensor RasterizeMeshesBackwardCuda( + const torch::Tensor& face_verts, + const torch::Tensor& pix_to_face, + const torch::Tensor& grad_zbuf, + const torch::Tensor& grad_bary, + const torch::Tensor& grad_dists, + const bool perspective_correct, + const bool clip_barycentric_coords); +#endif + +// Args: +// face_verts: float32 Tensor of shape (F, 3, 3) (from forward pass) giving +// (packed) vertex positions for faces in all the meshes in +// the batch. +// pix_to_face: int64 tensor of shape (N, H, W, K) giving the face index of +// each of the closest faces to the pixel in the rasterized +// image, or -1 for pixels that are not covered by any face. +// grad_zbuf: Tensor of shape (N, H, W, K) giving upstream gradients +// d(loss)/d(zbuf) of the zbuf tensor from the forward pass. +// grad_bary: Tensor of shape (N, H, W, K, 3) giving upstream gradients +// d(loss)/d(bary) of the barycentric_coords tensor returned by +// the forward pass. +// grad_dists: Tensor of shape (N, H, W, K) giving upstream gradients +// d(loss)/d(dists) of the dists tensor from the forward pass. +// perspective_correct: Whether to apply perspective correction when +// computing barycentric coordinates. If this is True, +// then this function returns world-space barycentric +// coordinates for each pixel; if this is False then +// this function instead returns screen-space +// barycentric coordinates for each pixel. +// clip_barycentric_coords: Whether, after any perspective correction +// is applied but before the depth is calculated (e.g. for +// z clipping), to "correct" a location outside the face (i.e. with +// a negative barycentric coordinate) to a position on the edge of the +// face. +// +// Returns: +// grad_face_verts: float32 Tensor of shape (F, 3, 3) giving downstream +// gradients for the face vertices. +torch::Tensor RasterizeMeshesBackward( + const torch::Tensor& face_verts, + const torch::Tensor& pix_to_face, + const torch::Tensor& grad_zbuf, + const torch::Tensor& grad_bary, + const torch::Tensor& grad_dists, + const bool perspective_correct, + const bool clip_barycentric_coords) { + if (face_verts.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(face_verts); + CHECK_CUDA(pix_to_face); + CHECK_CUDA(grad_zbuf); + CHECK_CUDA(grad_bary); + CHECK_CUDA(grad_dists); + return RasterizeMeshesBackwardCuda( + face_verts, + pix_to_face, + grad_zbuf, + grad_bary, + grad_dists, + perspective_correct, + clip_barycentric_coords); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } else { + return RasterizeMeshesBackwardCpu( + face_verts, + pix_to_face, + grad_zbuf, + grad_bary, + grad_dists, + perspective_correct, + clip_barycentric_coords); + } +} + +// **************************************************************************** +// * COARSE RASTERIZATION * +// **************************************************************************** + +// RasterizeMeshesCoarseCuda in rasterize_coarse/rasterize_coarse.h + +torch::Tensor RasterizeMeshesCoarseCpu( + const torch::Tensor& face_verts, + const at::Tensor& mesh_to_face_first_idx, + const at::Tensor& num_faces_per_mesh, + const std::tuple image_size, + const float blur_radius, + const int bin_size, + const int max_faces_per_bin); + +// Args: +// face_verts: Tensor of shape (F, 3, 3) giving (packed) vertex positions for +// faces in all the meshes in the batch. Concretely, +// face_verts[f, i] = [x, y, z] gives the coordinates for the +// ith vertex of the fth face. These vertices are expected to be +// in NDC coordinates in the range [-1, 1]. +// mesh_to_face_first_idx: LongTensor of shape (N) giving the index in +// faces_verts of the first face in each mesh in +// the batch where N is the batch size. +// num_faces_per_mesh: LongTensor of shape (N) giving the number of faces +// for each mesh in the batch. +// image_size: Tuple (H, W) giving the size in pixels of the output +// image to be rasterized. +// blur_radius: float distance in NDC coordinates uses to expand the face +// bounding boxes for the rasterization. Set to 0.0 if no blur +// is required. +// bin_size: Size of each bin within the image (in pixels) +// max_faces_per_bin: Maximum number of faces to count in each bin. +// +// Returns: +// bin_face_idxs: Tensor of shape (N, num_bins, num_bins, K) giving the +// indices of faces that fall into each bin. + +torch::Tensor RasterizeMeshesCoarse( + const torch::Tensor& face_verts, + const torch::Tensor& mesh_to_face_first_idx, + const torch::Tensor& num_faces_per_mesh, + const std::tuple image_size, + const float blur_radius, + const int bin_size, + const int max_faces_per_bin) { + if (face_verts.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(face_verts); + CHECK_CUDA(mesh_to_face_first_idx); + CHECK_CUDA(num_faces_per_mesh); + return RasterizeMeshesCoarseCuda( + face_verts, + mesh_to_face_first_idx, + num_faces_per_mesh, + image_size, + blur_radius, + bin_size, + max_faces_per_bin); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } else { + return RasterizeMeshesCoarseCpu( + face_verts, + mesh_to_face_first_idx, + num_faces_per_mesh, + image_size, + blur_radius, + bin_size, + max_faces_per_bin); + } +} + +// **************************************************************************** +// * FINE RASTERIZATION * +// **************************************************************************** + +#ifdef WITH_CUDA +std::tuple +RasterizeMeshesFineCuda( + const torch::Tensor& face_verts, + const torch::Tensor& bin_faces, + const torch::Tensor& clipped_faces_neighbor_idx, + const std::tuple image_size, + const float blur_radius, + const int bin_size, + const int faces_per_pixel, + const bool perspective_correct, + const bool clip_barycentric_coords, + const bool cull_backfaces); +#endif +// Args: +// face_verts: Tensor of shape (F, 3, 3) giving (packed) vertex positions for +// faces in all the meshes in the batch. Concretely, +// face_verts[f, i] = [x, y, z] gives the coordinates for the +// ith vertex of the fth face. These vertices are expected to be +// in NDC coordinates in the range [-1, 1]. +// bin_faces: int32 Tensor of shape (N, B, B, M) giving the indices of faces +// that fall into each bin (output from coarse rasterization). +// clipped_faces_neighbor_idx: LongTensor of shape (F,) giving the +// index of the neighboring face for each face which was clipped to a +// quadrilateral and then divided into two triangles. +// e.g. for a face f partially behind the image plane which is split into +// two triangles (t1, t2): clipped_faces_neighbor_idx[t1_idx] = t2_idx +// Faces which are not clipped and subdivided are set to -1. +// image_size: Tuple (H, W) giving the size in pixels of the output +// image to be rasterized. +// blur_radius: float distance in NDC coordinates uses to expand the face +// bounding boxes for the rasterization. Set to 0.0 if no blur +// is required. +// bin_size: Size of each bin within the image (in pixels) +// faces_per_pixel: the number of closeset faces to rasterize per pixel. +// perspective_correct: Whether to apply perspective correction when +// computing barycentric coordinates. If this is True, +// then this function returns world-space barycentric +// coordinates for each pixel; if this is False then +// this function instead returns screen-space +// barycentric coordinates for each pixel. +// clip_barycentric_coords: Whether, after any perspective correction +// is applied but before the depth is calculated (e.g. for +// z clipping), to "correct" a location outside the face (i.e. with +// a negative barycentric coordinate) to a position on the edge of the +// face. +// cull_backfaces: Bool, Whether to only rasterize mesh faces which are +// visible to the camera. This assumes that vertices of +// front-facing triangles are ordered in an anti-clockwise +// fashion, and triangles that face away from the camera are +// in a clockwise order relative to the current view +// direction. NOTE: This will only work if the mesh faces are +// consistently defined with counter-clockwise ordering when +// viewed from the outside. +// +// Returns (same as rasterize_meshes): +// A 4 element tuple of: +// pix_to_face: int64 tensor of shape (N, H, W, K) giving the face index of +// each of the closest faces to the pixel in the rasterized +// image, or -1 for pixels that are not covered by any face. +// zbuf: float32 Tensor of shape (N, H, W, K) giving the depth of each of +// the closest faces for each pixel. +// barycentric_coords: float tensor of shape (N, H, W, K, 3) giving +// barycentric coordinates of the pixel with respect to +// each of the closest faces along the z axis, padded +// with -1 for pixels hit by fewer than +// faces_per_pixel faces. +// dists: float tensor of shape (N, H, W, K) giving the euclidean distance +// in the (NDC) x/y plane between each pixel and its K closest +// faces along the z axis padded with -1 for pixels hit by fewer than +// faces_per_pixel faces. +std::tuple +RasterizeMeshesFine( + const torch::Tensor& face_verts, + const torch::Tensor& bin_faces, + const torch::Tensor& clipped_faces_neighbor_idx, + const std::tuple image_size, + const float blur_radius, + const int bin_size, + const int faces_per_pixel, + const bool perspective_correct, + const bool clip_barycentric_coords, + const bool cull_backfaces) { + if (face_verts.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(face_verts); + CHECK_CUDA(bin_faces); + return RasterizeMeshesFineCuda( + face_verts, + bin_faces, + clipped_faces_neighbor_idx, + image_size, + blur_radius, + bin_size, + faces_per_pixel, + perspective_correct, + clip_barycentric_coords, + cull_backfaces); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } else { + AT_ERROR("NOT IMPLEMENTED"); + } +} + +// **************************************************************************** +// * MAIN ENTRY POINT * +// **************************************************************************** + +// This is the main entry point for the forward pass of the mesh rasterizer; +// it uses either naive or coarse-to-fine rasterization based on bin_size. +// +// Args: +// face_verts: Tensor of shape (F, 3, 3) giving (packed) vertex positions for +// faces in all the meshes in the batch. Concretely, +// face_verts[f, i] = [x, y, z] gives the coordinates for the +// ith vertex of the fth face. These vertices are expected to be +// in NDC coordinates in the range [-1, 1]. +// mesh_to_face_first_idx: LongTensor of shape (N) giving the index in +// faces_verts of the first face in each mesh in +// the batch where N is the batch size. +// num_faces_per_mesh: LongTensor of shape (N) giving the number of faces +// for each mesh in the batch. +// clipped_faces_neighbor_idx: LongTensor of shape (F,) giving the +// index of the neighboring face for each face which was clipped to a +// quadrilateral and then divided into two triangles. +// e.g. for a face f partially behind the image plane which is split into +// two triangles (t1, t2): clipped_faces_neighbor_idx[t1_idx] = t2_idx +// Faces which are not clipped and subdivided are set to -1. +// image_size: Tuple (H, W) giving the size in pixels of the output +// image to be rasterized. +// blur_radius: float distance in NDC coordinates uses to expand the face +// bounding boxes for the rasterization. Set to 0.0 if no blur +// is required. +// faces_per_pixel: the number of closeset faces to rasterize per pixel. +// bin_size: Bin size (in pixels) for coarse-to-fine rasterization. Setting +// bin_size=0 uses naive rasterization instead. +// max_faces_per_bin: The maximum number of faces allowed to fall into each +// bin when using coarse-to-fine rasterization. +// perspective_correct: Whether to apply perspective correction when +// computing barycentric coordinates. If this is True, +// then this function returns world-space barycentric +// coordinates for each pixel; if this is False then +// this function instead returns screen-space +// barycentric coordinates for each pixel. +// clip_barycentric_coords: Whether, after any perspective correction +// is applied but before the depth is calculated (e.g. for +// z clipping), to "correct" a location outside the face (i.e. with +// a negative barycentric coordinate) to a position on the edge of the +// face. +// cull_backfaces: Bool, Whether to only rasterize mesh faces which are +// visible to the camera. This assumes that vertices of +// front-facing triangles are ordered in an anti-clockwise +// fashion, and triangles that face away from the camera are +// in a clockwise order relative to the current view +// direction. NOTE: This will only work if the mesh faces are +// consistently defined with counter-clockwise ordering when +// viewed from the outside. +// +// Returns: +// A 4 element tuple of: +// pix_to_face: int64 tensor of shape (N, H, W, K) giving the face index of +// each of the closest faces to the pixel in the rasterized +// image, or -1 for pixels that are not covered by any face. +// zbuf: float32 Tensor of shape (N, H, W, K) giving the depth of each of +// the closest faces for each pixel. +// barycentric_coords: float tensor of shape (N, H, W, K, 3) giving +// barycentric coordinates of the pixel with respect to +// each of the closest faces along the z axis, padded +// with -1 for pixels hit by fewer than +// faces_per_pixel faces. +// dists: float tensor of shape (N, H, W, K) giving the euclidean distance +// in the (NDC) x/y plane between each pixel and its K closest +// faces along the z axis padded with -1 for pixels hit by fewer than +// faces_per_pixel faces. +std::tuple +RasterizeMeshes( + const torch::Tensor& face_verts, + const torch::Tensor& mesh_to_face_first_idx, + const torch::Tensor& num_faces_per_mesh, + const torch::Tensor& clipped_faces_neighbor_idx, + const std::tuple image_size, + const float blur_radius, + const int faces_per_pixel, + const int bin_size, + const int max_faces_per_bin, + const bool perspective_correct, + const bool clip_barycentric_coords, + const bool cull_backfaces) { + if (bin_size > 0 && max_faces_per_bin > 0) { + // Use coarse-to-fine rasterization + at::Tensor bin_faces = RasterizeMeshesCoarse( + face_verts, + mesh_to_face_first_idx, + num_faces_per_mesh, + image_size, + blur_radius, + bin_size, + max_faces_per_bin); + return RasterizeMeshesFine( + face_verts, + bin_faces, + clipped_faces_neighbor_idx, + image_size, + blur_radius, + bin_size, + faces_per_pixel, + perspective_correct, + clip_barycentric_coords, + cull_backfaces); + } else { + // Use the naive per-pixel implementation + return RasterizeMeshesNaive( + face_verts, + mesh_to_face_first_idx, + num_faces_per_mesh, + clipped_faces_neighbor_idx, + image_size, + blur_radius, + faces_per_pixel, + perspective_correct, + clip_barycentric_coords, + cull_backfaces); + } +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_meshes/rasterize_meshes_cpu.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_meshes/rasterize_meshes_cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..210df55e43de5602c3c80b05e0ff8b9d7e59253a --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_meshes/rasterize_meshes_cpu.cpp @@ -0,0 +1,640 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include +#include +#include +#include +#include "ATen/core/TensorAccessor.h" +#include "rasterize_points/rasterization_utils.h" +#include "utils/geometry_utils.h" +#include "utils/vec2.h" +#include "utils/vec3.h" + +// Get (x, y, z) values for vertex from (3, 3) tensor face. +template +auto ExtractVerts(const Face& face, const int vertex_index) { + return std::make_tuple( + face[vertex_index][0], face[vertex_index][1], face[vertex_index][2]); +} + +// Compute min/max x/y for each face. +auto ComputeFaceBoundingBoxes(const torch::Tensor& face_verts) { + const int total_F = face_verts.size(0); + auto float_opts = face_verts.options().dtype(torch::kFloat32); + auto face_verts_a = face_verts.accessor(); + torch::Tensor face_bboxes = torch::full({total_F, 6}, -2.0, float_opts); + + // Loop through all the faces + for (int f = 0; f < total_F; ++f) { + const auto& face = face_verts_a[f]; + float x0, x1, x2, y0, y1, y2, z0, z1, z2; + std::tie(x0, y0, z0) = ExtractVerts(face, 0); + std::tie(x1, y1, z1) = ExtractVerts(face, 1); + std::tie(x2, y2, z2) = ExtractVerts(face, 2); + + const float x_min = std::min(x0, std::min(x1, x2)); + const float y_min = std::min(y0, std::min(y1, y2)); + const float x_max = std::max(x0, std::max(x1, x2)); + const float y_max = std::max(y0, std::max(y1, y2)); + const float z_min = std::min(z0, std::min(z1, z2)); + const float z_max = std::max(z0, std::max(z1, z2)); + + face_bboxes[f][0] = x_min; + face_bboxes[f][1] = y_min; + face_bboxes[f][2] = x_max; + face_bboxes[f][3] = y_max; + face_bboxes[f][4] = z_min; + face_bboxes[f][5] = z_max; + } + + return face_bboxes; +} + +// Check if the point (px, py) lies inside the face bounding box face_bbox. +// Return true if the point is outside. +template +bool CheckPointOutsideBoundingBox( + const Face& face_bbox, + float blur_radius, + float px, + float py) { + // Read triangle bbox coordinates and expand by blur radius. + float x_min = face_bbox[0] - blur_radius; + float y_min = face_bbox[1] - blur_radius; + float x_max = face_bbox[2] + blur_radius; + float y_max = face_bbox[3] + blur_radius; + + // Faces with at least one vertex behind the camera won't render correctly + // and should be removed or clipped before calling the rasterizer + const bool z_invalid = face_bbox[4] < kEpsilon; + + // Check if the current point is within the triangle bounding box. + return (px > x_max || px < x_min || py > y_max || py < y_min || z_invalid); +} + +// Calculate areas of all faces. Returns a tensor of shape (total_faces, 1) +// where faces with zero area have value -1. +auto ComputeFaceAreas(const torch::Tensor& face_verts) { + const int total_F = face_verts.size(0); + auto float_opts = face_verts.options().dtype(torch::kFloat32); + auto face_verts_a = face_verts.accessor(); + torch::Tensor face_areas = torch::full({total_F}, -1, float_opts); + + // Loop through all the faces + for (int f = 0; f < total_F; ++f) { + const auto& face = face_verts_a[f]; + float x0, x1, x2, y0, y1, y2, z0, z1, z2; + std::tie(x0, y0, z0) = ExtractVerts(face, 0); + std::tie(x1, y1, z1) = ExtractVerts(face, 1); + std::tie(x2, y2, z2) = ExtractVerts(face, 2); + + const vec2 v0(x0, y0); + const vec2 v1(x1, y1); + const vec2 v2(x2, y2); + + const float face_area = EdgeFunctionForward(v0, v1, v2); + face_areas[f] = face_area; + } + + return face_areas; +} + +// Helper function to use with std::find_if to find the index of any +// values in the top k struct which match a given idx. +struct IsNeighbor { + IsNeighbor(int neighbor_idx) { + this->neighbor_idx = neighbor_idx; + } + bool operator()(std::tuple elem) { + return (std::get<1>(elem) == neighbor_idx); + } + int neighbor_idx; +}; + +namespace { +void RasterizeMeshesNaiveCpu_worker( + const int start_yi, + const int end_yi, + const torch::Tensor& mesh_to_face_first_idx, + const torch::Tensor& num_faces_per_mesh, + const float blur_radius, + const bool perspective_correct, + const bool clip_barycentric_coords, + const bool cull_backfaces, + const int32_t N, + const int H, + const int W, + const int K, + at::TensorAccessor& face_verts_a, + at::TensorAccessor& face_areas_a, + at::TensorAccessor& face_bboxes_a, + at::TensorAccessor& neighbor_idx_a, + at::TensorAccessor& zbuf_a, + at::TensorAccessor& face_idxs_a, + at::TensorAccessor& pix_dists_a, + at::TensorAccessor& barycentric_coords_a) { + for (int n = 0; n < N; ++n) { + // Loop through each mesh in the batch. + // Get the start index of the faces in faces_packed and the num faces + // in the mesh to avoid having to loop through all the faces. + const int face_start_idx = mesh_to_face_first_idx[n].item().to(); + const int face_stop_idx = + (face_start_idx + num_faces_per_mesh[n].item().to()); + + // Iterate through the horizontal lines of the image from top to bottom. + for (int yi = start_yi; yi < end_yi; ++yi) { + // Reverse the order of yi so that +Y is pointing upwards in the image. + const int yidx = H - 1 - yi; + + // Y coordinate of the top of the pixel. + const float yf = PixToNonSquareNdc(yidx, H, W); + // Iterate through pixels on this horizontal line, left to right. + for (int xi = 0; xi < W; ++xi) { + // Reverse the order of xi so that +X is pointing to the left in the + // image. + const int xidx = W - 1 - xi; + + // X coordinate of the left of the pixel. + const float xf = PixToNonSquareNdc(xidx, W, H); + + // Use a deque to hold values: + // (z, idx, r, bary.x, bary.y. bary.z) + // Sort the deque as needed to mimic a priority queue. + std::deque> q; + + // Loop through the faces in the mesh. + for (int f = face_start_idx; f < face_stop_idx; ++f) { + // Get coordinates of three face vertices. + const auto& face = face_verts_a[f]; + float x0, x1, x2, y0, y1, y2, z0, z1, z2; + std::tie(x0, y0, z0) = ExtractVerts(face, 0); + std::tie(x1, y1, z1) = ExtractVerts(face, 1); + std::tie(x2, y2, z2) = ExtractVerts(face, 2); + + const vec2 v0(x0, y0); + const vec2 v1(x1, y1); + const vec2 v2(x2, y2); + + const float face_area = face_areas_a[f]; + const bool back_face = face_area < 0.0; + // Check if the face is visible to the camera. + if (cull_backfaces && back_face) { + continue; + } + // Skip faces with zero area. + if (face_area <= kEpsilon && face_area >= -1.0f * kEpsilon) { + continue; + } + + // Skip if point is outside the face bounding box. + const auto face_bbox = face_bboxes_a[f]; + const bool outside_bbox = CheckPointOutsideBoundingBox( + face_bbox, std::sqrt(blur_radius), xf, yf); + if (outside_bbox) { + continue; + } + + // Compute barycentric coordinates and use this to get the + // depth of the point on the triangle. + const vec2 pxy(xf, yf); + const vec3 bary0 = + BarycentricCoordinatesForward(pxy, v0, v1, v2); + const vec3 bary = !perspective_correct + ? bary0 + : BarycentricPerspectiveCorrectionForward(bary0, z0, z1, z2); + + const vec3 bary_clip = + !clip_barycentric_coords ? bary : BarycentricClipForward(bary); + + // Use barycentric coordinates to get the depth of the current pixel + const float pz = + (bary_clip.x * z0 + bary_clip.y * z1 + bary_clip.z * z2); + + if (pz < 0) { + continue; // Point is behind the image plane so ignore. + } + + // Compute squared distance of the point to the triangle. + const float dist = PointTriangleDistanceForward(pxy, v0, v1, v2); + + // Use the bary coordinates to determine if the point is + // inside the face. + const bool inside = bary.x > 0.0f && bary.y > 0.0f && bary.z > 0.0f; + + // If the point is inside the triangle then signed_dist + // is negative. + const float signed_dist = inside ? -dist : dist; + + // Check if pixel is outside blur region + if (!inside && dist >= blur_radius) { + continue; + } + + // Handle the case where a face (f) partially behind the image plane + // is clipped to a quadrilateral and then split into two faces (t1, + // t2). In this case we: + // 1. Find the index of the neighbor (e.g. for t1 need index of t2) + // 2. Check if the neighbor (t2) is already in the top K faces + // 3. If yes, compare the distance of the pixel to t1 with the + // distance to t2. + // 4. If dist_t1 < dist_t2, overwrite the values for t2 in the top K + // faces. + const int neighbor_idx = neighbor_idx_a[f]; + int idx_top_k = -1; + + // Check if neighboring face is already in the top K. + if (neighbor_idx != -1) { + const auto it = + std::find_if(q.begin(), q.end(), IsNeighbor(neighbor_idx)); + // Get the index of the element from the iterator + idx_top_k = (it != q.end()) ? it - q.begin() : idx_top_k; + } + + // If idx_top_k idx is not -1 then it is in the top K struct. + if (idx_top_k != -1) { + // If dist of current face is less than neighbor, overwrite + // the neighbor face values in the top K struct. + const auto neighbor = q[idx_top_k]; + const float dist_neighbor = std::abs(std::get<2>(neighbor)); + if (dist < dist_neighbor) { + // Overwrite the neighbor face values. + q[idx_top_k] = std::make_tuple( + pz, f, signed_dist, bary_clip.x, bary_clip.y, bary_clip.z); + } + } else { + // Handle as a normal face. + // The current pixel lies inside the current face. + // Add at the end of the deque. + q.emplace_back( + pz, f, signed_dist, bary_clip.x, bary_clip.y, bary_clip.z); + } + + // Sort the deque inplace based on the z distance + // to mimic using a priority queue. + std::sort(q.begin(), q.end()); + if (static_cast(q.size()) > K) { + // remove the last value + q.pop_back(); + } + } + while (!q.empty()) { + // Loop through and add values to the output tensors + auto t = q.back(); + q.pop_back(); + const int i = q.size(); + zbuf_a[n][yi][xi][i] = std::get<0>(t); + face_idxs_a[n][yi][xi][i] = std::get<1>(t); + pix_dists_a[n][yi][xi][i] = std::get<2>(t); + barycentric_coords_a[n][yi][xi][i][0] = std::get<3>(t); + barycentric_coords_a[n][yi][xi][i][1] = std::get<4>(t); + barycentric_coords_a[n][yi][xi][i][2] = std::get<5>(t); + } + } + } + } +} +} // namespace + +std::tuple +RasterizeMeshesNaiveCpu( + const torch::Tensor& face_verts, + const torch::Tensor& mesh_to_face_first_idx, + const torch::Tensor& num_faces_per_mesh, + const torch::Tensor& clipped_faces_neighbor_idx, + const std::tuple image_size, + const float blur_radius, + const int faces_per_pixel, + const bool perspective_correct, + const bool clip_barycentric_coords, + const bool cull_backfaces) { + if (face_verts.ndimension() != 3 || face_verts.size(1) != 3 || + face_verts.size(2) != 3) { + AT_ERROR("face_verts must have dimensions (num_faces, 3, 3)"); + } + if (num_faces_per_mesh.size(0) != mesh_to_face_first_idx.size(0)) { + AT_ERROR( + "num_faces_per_mesh must have save size first dimension as mesh_to_face_first_idx"); + } + + const int32_t N = mesh_to_face_first_idx.size(0); // batch_size. + const int H = std::get<0>(image_size); + const int W = std::get<1>(image_size); + const int K = faces_per_pixel; + + auto long_opts = num_faces_per_mesh.options().dtype(torch::kInt64); + auto float_opts = face_verts.options().dtype(torch::kFloat32); + + // Initialize output tensors. + torch::Tensor face_idxs = torch::full({N, H, W, K}, -1, long_opts); + torch::Tensor zbuf = torch::full({N, H, W, K}, -1, float_opts); + torch::Tensor pix_dists = torch::full({N, H, W, K}, -1, float_opts); + torch::Tensor barycentric_coords = + torch::full({N, H, W, K, 3}, -1, float_opts); + + auto face_verts_a = face_verts.accessor(); + auto face_idxs_a = face_idxs.accessor(); + auto zbuf_a = zbuf.accessor(); + auto pix_dists_a = pix_dists.accessor(); + auto barycentric_coords_a = barycentric_coords.accessor(); + auto neighbor_idx_a = clipped_faces_neighbor_idx.accessor(); + + auto face_bboxes = ComputeFaceBoundingBoxes(face_verts); + auto face_bboxes_a = face_bboxes.accessor(); + auto face_areas = ComputeFaceAreas(face_verts); + auto face_areas_a = face_areas.accessor(); + + const int64_t n_threads = at::get_num_threads(); + std::vector threads; + threads.reserve(n_threads); + const int chunk_size = 1 + (H - 1) / n_threads; + int start_yi = 0; + for (int iThread = 0; iThread < n_threads; ++iThread) { + const int64_t end_yi = std::min(start_yi + chunk_size, H); + threads.emplace_back( + RasterizeMeshesNaiveCpu_worker, + start_yi, + end_yi, + mesh_to_face_first_idx, + num_faces_per_mesh, + blur_radius, + perspective_correct, + clip_barycentric_coords, + cull_backfaces, + N, + H, + W, + K, + std::ref(face_verts_a), + std::ref(face_areas_a), + std::ref(face_bboxes_a), + std::ref(neighbor_idx_a), + std::ref(zbuf_a), + std::ref(face_idxs_a), + std::ref(pix_dists_a), + std::ref(barycentric_coords_a)); + start_yi += chunk_size; + } + for (auto&& thread : threads) { + thread.join(); + } + + return std::make_tuple(face_idxs, zbuf, barycentric_coords, pix_dists); +} + +torch::Tensor RasterizeMeshesBackwardCpu( + const torch::Tensor& face_verts, // (F, 3, 3) + const torch::Tensor& pix_to_face, // (N, H, W, K) + const torch::Tensor& grad_zbuf, // (N, H, W, K) + const torch::Tensor& grad_bary, // (N, H, W, K, 3) + const torch::Tensor& grad_dists, // (N, H, W, K) + const bool perspective_correct, + const bool clip_barycentric_coords) { + const int F = face_verts.size(0); + const int N = pix_to_face.size(0); + const int H = pix_to_face.size(1); + const int W = pix_to_face.size(2); + const int K = pix_to_face.size(3); + + torch::Tensor grad_face_verts = torch::zeros({F, 3, 3}, face_verts.options()); + auto face_verts_a = face_verts.accessor(); + auto pix_to_face_a = pix_to_face.accessor(); + auto grad_dists_a = grad_dists.accessor(); + auto grad_zbuf_a = grad_zbuf.accessor(); + auto grad_bary_a = grad_bary.accessor(); + + for (int n = 0; n < N; ++n) { + // Iterate through the horizontal lines of the image from top to bottom. + for (int y = 0; y < H; ++y) { + // Reverse the order of yi so that +Y is pointing upwards in the image. + const int yidx = H - 1 - y; + + // Y coordinate of the top of the pixel. + const float yf = PixToNonSquareNdc(yidx, H, W); + // Iterate through pixels on this horizontal line, left to right. + for (int x = 0; x < W; ++x) { + // Reverse the order of xi so that +X is pointing to the left in the + // image. + const int xidx = W - 1 - x; + + // X coordinate of the left of the pixel. + const float xf = PixToNonSquareNdc(xidx, W, H); + const vec2 pxy(xf, yf); + + // Iterate through the faces that hit this pixel. + for (int k = 0; k < K; ++k) { + // Get face index from forward pass output. + const int f = pix_to_face_a[n][y][x][k]; + if (f < 0) { + continue; // padded face. + } + // Get coordinates of the three face vertices. + const auto face_verts_f = face_verts_a[f]; + const float x0 = face_verts_f[0][0]; + const float y0 = face_verts_f[0][1]; + const float z0 = face_verts_f[0][2]; + const float x1 = face_verts_f[1][0]; + const float y1 = face_verts_f[1][1]; + const float z1 = face_verts_f[1][2]; + const float x2 = face_verts_f[2][0]; + const float y2 = face_verts_f[2][1]; + const float z2 = face_verts_f[2][2]; + const vec2 v0xy(x0, y0); + const vec2 v1xy(x1, y1); + const vec2 v2xy(x2, y2); + + // Get upstream gradients for the face. + const float grad_dist_upstream = grad_dists_a[n][y][x][k]; + const float grad_zbuf_upstream = grad_zbuf_a[n][y][x][k]; + const auto grad_bary_upstream_w012 = grad_bary_a[n][y][x][k]; + const float grad_bary_upstream_w0 = grad_bary_upstream_w012[0]; + const float grad_bary_upstream_w1 = grad_bary_upstream_w012[1]; + const float grad_bary_upstream_w2 = grad_bary_upstream_w012[2]; + const vec3 grad_bary_upstream( + grad_bary_upstream_w0, + grad_bary_upstream_w1, + grad_bary_upstream_w2); + + const vec3 bary0 = + BarycentricCoordinatesForward(pxy, v0xy, v1xy, v2xy); + const vec3 bary = !perspective_correct + ? bary0 + : BarycentricPerspectiveCorrectionForward(bary0, z0, z1, z2); + const vec3 bary_clip = + !clip_barycentric_coords ? bary : BarycentricClipForward(bary); + + // Distances inside the face are negative so get the + // correct sign to apply to the upstream gradient. + const bool inside = bary.x > 0.0f && bary.y > 0.0f && bary.z > 0.0f; + const float sign = inside ? -1.0f : 1.0f; + + const auto grad_dist_f = PointTriangleDistanceBackward( + pxy, v0xy, v1xy, v2xy, sign * grad_dist_upstream); + const auto ddist_d_v0 = std::get<1>(grad_dist_f); + const auto ddist_d_v1 = std::get<2>(grad_dist_f); + const auto ddist_d_v2 = std::get<3>(grad_dist_f); + + // Upstream gradient for barycentric coords from zbuf calculation: + // zbuf = bary_w0 * z0 + bary_w1 * z1 + bary_w2 * z2 + // Therefore + // d_zbuf/d_bary_w0 = z0 + // d_zbuf/d_bary_w1 = z1 + // d_zbuf/d_bary_w2 = z2 + const vec3 d_zbuf_d_baryclip(z0, z1, z2); + + // Total upstream barycentric gradients are the sum of + // external upstream gradients and contribution from zbuf. + const vec3 grad_bary_f_sum = + (grad_bary_upstream + grad_zbuf_upstream * d_zbuf_d_baryclip); + + vec3 grad_bary0 = grad_bary_f_sum; + + if (clip_barycentric_coords) { + grad_bary0 = BarycentricClipBackward(bary, grad_bary0); + } + + if (perspective_correct) { + auto perspective_grads = BarycentricPerspectiveCorrectionBackward( + bary0, z0, z1, z2, grad_bary0); + grad_bary0 = std::get<0>(perspective_grads); + grad_face_verts[f][0][2] += std::get<1>(perspective_grads); + grad_face_verts[f][1][2] += std::get<2>(perspective_grads); + grad_face_verts[f][2][2] += std::get<3>(perspective_grads); + } + + auto grad_bary_f = + BarycentricCoordsBackward(pxy, v0xy, v1xy, v2xy, grad_bary0); + const vec2 dbary_d_v0 = std::get<1>(grad_bary_f); + const vec2 dbary_d_v1 = std::get<2>(grad_bary_f); + const vec2 dbary_d_v2 = std::get<3>(grad_bary_f); + + // Update output gradient buffer. + grad_face_verts[f][0][0] += dbary_d_v0.x + ddist_d_v0.x; + grad_face_verts[f][0][1] += dbary_d_v0.y + ddist_d_v0.y; + grad_face_verts[f][0][2] += grad_zbuf_upstream * bary_clip.x; + grad_face_verts[f][1][0] += dbary_d_v1.x + ddist_d_v1.x; + grad_face_verts[f][1][1] += dbary_d_v1.y + ddist_d_v1.y; + grad_face_verts[f][1][2] += grad_zbuf_upstream * bary_clip.y; + grad_face_verts[f][2][0] += dbary_d_v2.x + ddist_d_v2.x; + grad_face_verts[f][2][1] += dbary_d_v2.y + ddist_d_v2.y; + grad_face_verts[f][2][2] += grad_zbuf_upstream * bary_clip.z; + } + } + } + } + return grad_face_verts; +} + +torch::Tensor RasterizeMeshesCoarseCpu( + const torch::Tensor& face_verts, + const torch::Tensor& mesh_to_face_first_idx, + const torch::Tensor& num_faces_per_mesh, + const std::tuple image_size, + const float blur_radius, + const int bin_size, + const int max_faces_per_bin) { + if (face_verts.ndimension() != 3 || face_verts.size(1) != 3 || + face_verts.size(2) != 3) { + AT_ERROR("face_verts must have dimensions (num_faces, 3, 3)"); + } + if (num_faces_per_mesh.ndimension() != 1) { + AT_ERROR("num_faces_per_mesh can only have one dimension"); + } + + const int N = num_faces_per_mesh.size(0); // batch size. + const int M = max_faces_per_bin; + + const float H = std::get<0>(image_size); + const float W = std::get<1>(image_size); + + // Integer division round up. + const int BH = 1 + (H - 1) / bin_size; + const int BW = 1 + (W - 1) / bin_size; + + auto opts = num_faces_per_mesh.options().dtype(torch::kInt32); + torch::Tensor faces_per_bin = torch::zeros({N, BH, BW}, opts); + torch::Tensor bin_faces = torch::full({N, BH, BW, M}, -1, opts); + auto bin_faces_a = bin_faces.accessor(); + + // Precompute all face bounding boxes. + auto face_bboxes = ComputeFaceBoundingBoxes(face_verts); + auto face_bboxes_a = face_bboxes.accessor(); + + const float ndc_x_range = NonSquareNdcRange(W, H); + const float pixel_width_x = ndc_x_range / W; + const float bin_width_x = pixel_width_x * bin_size; + + const float ndc_y_range = NonSquareNdcRange(H, W); + const float pixel_width_y = ndc_y_range / H; + const float bin_width_y = pixel_width_y * bin_size; + + // Iterate through the meshes in the batch. + for (int n = 0; n < N; ++n) { + const int face_start_idx = mesh_to_face_first_idx[n].item().to(); + const int face_stop_idx = + (face_start_idx + num_faces_per_mesh[n].item().to()); + + float bin_y_min = -1.0f; + float bin_y_max = bin_y_min + bin_width_y; + + // Iterate through the horizontal bins from top to bottom. + for (int by = 0; by < BH; ++by) { + float bin_x_min = -1.0f; + float bin_x_max = bin_x_min + bin_width_x; + + // Iterate through bins on this horizontal line, left to right. + for (int bx = 0; bx < BW; ++bx) { + int32_t faces_hit = 0; + + for (int32_t f = face_start_idx; f < face_stop_idx; ++f) { + // Get bounding box and expand by blur radius. + float face_x_min = face_bboxes_a[f][0] - std::sqrt(blur_radius); + float face_y_min = face_bboxes_a[f][1] - std::sqrt(blur_radius); + float face_x_max = face_bboxes_a[f][2] + std::sqrt(blur_radius); + float face_y_max = face_bboxes_a[f][3] + std::sqrt(blur_radius); + float face_z_min = face_bboxes_a[f][4]; + + // Faces with at least one vertex behind the camera won't render + // correctly and should be removed or clipped before calling the + // rasterizer + if (face_z_min < kEpsilon) { + continue; + } + + // Use a half-open interval so that faces exactly on the + // boundary between bins will fall into exactly one bin. + bool x_overlap = + (face_x_min <= bin_x_max) && (bin_x_min < face_x_max); + bool y_overlap = + (face_y_min <= bin_y_max) && (bin_y_min < face_y_max); + + if (x_overlap && y_overlap) { + // Got too many faces for this bin, so throw an error. + if (faces_hit >= max_faces_per_bin) { + AT_ERROR("Got too many faces per bin"); + } + // The current point falls in the current bin, so + // record it. + bin_faces_a[n][by][bx][faces_hit] = f; + faces_hit++; + } + } + + // Shift the bin to the right for the next loop iteration + bin_x_min = bin_x_max; + bin_x_max = bin_x_min + bin_width_x; + } + // Shift the bin down for the next loop iteration + bin_y_min = bin_y_max; + bin_y_max = bin_y_min + bin_width_y; + } + } + return bin_faces; +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_points/rasterization_utils.cuh b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_points/rasterization_utils.cuh new file mode 100644 index 0000000000000000000000000000000000000000..600d7a1afff8a5089ad2d8d8023e195bcaab480c --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_points/rasterization_utils.cuh @@ -0,0 +1,66 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +// The default value of the NDC range is [-1, 1], however in the case that +// H != W, the NDC range is set such that the shorter side has range [-1, 1] and +// the longer side is scaled by the ratio of H:W. S1 is the dimension for which +// the NDC range is calculated and S2 is the other image dimension. +// e.g. to get the NDC x range S1 = W and S2 = H +__device__ inline float NonSquareNdcRange(int S1, int S2) { + float range = 2.0f; + if (S1 > S2) { + // First multiply S1 by float range so that division results + // in a float value. + range = (S1 * range) / S2; + } + return range; +} + +// Given a pixel coordinate 0 <= i < S1, convert it to a normalized device +// coordinates. We divide the NDC range into S1 evenly-sized +// pixels, and assume that each pixel falls in the *center* of its range. +// The default value of the NDC range is [-1, 1], however in the case that +// H != W, the NDC range is set such that the shorter side has range [-1, 1] and +// the longer side is scaled by the ratio of H:W. The dimension of i should be +// S1 and the other image dimension is S2 For example, to get the x and y NDC +// coordinates or a given pixel i: +// x = PixToNonSquareNdc(i, W, H) +// y = PixToNonSquareNdc(i, H, W) +__device__ inline float PixToNonSquareNdc(int i, int S1, int S2) { + float range = NonSquareNdcRange(S1, S2); + // NDC: offset + (i * pixel_width + half_pixel_width) + // The NDC range is [-range/2, range/2]. + float offset = (range / 2.0f); + return -offset + (range * i + offset) / S1; +} + +// The maximum number of points per pixel that we can return. Since we use +// thread-local arrays to hold and sort points, the maximum size of the array +// needs to be known at compile time. There might be some fancy template magic +// we could use to make this more dynamic, but for now just fix a constant. +// TODO: is 8 enough? Would increasing have performance considerations? +const int32_t kMaxPointsPerPixel = 150; + +const int32_t kMaxItemsPerBin = 22; + +template +__device__ inline void BubbleSort(T* arr, int n) { + // Bubble sort. We only use it for tiny thread-local arrays (n < 8); in this + // regime we care more about warp divergence than computational complexity. + for (int i = 0; i < n - 1; ++i) { + for (int j = 0; j < n - i - 1; ++j) { + if (arr[j + 1] < arr[j]) { + T temp = arr[j]; + arr[j] = arr[j + 1]; + arr[j + 1] = temp; + } + } + } +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_points/rasterization_utils.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_points/rasterization_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..6980afc4a49962760283c46ac415faa5ab76ce8e --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_points/rasterization_utils.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +// The default value of the NDC range is [-1, 1], however in the case that +// H != W, the NDC range is set such that the shorter side has range [-1, 1] and +// the longer side is scaled by the ratio of H:W. S1 is the dimension for which +// the NDC range is calculated and S2 is the other image dimension. +// e.g. to get the NDC x range S1 = W and S2 = H +inline float NonSquareNdcRange(int S1, int S2) { + float range = 2.0f; + if (S1 > S2) { + range = (S1 * range) / S2; + } + return range; +} + +// Given a pixel coordinate 0 <= i < S1, convert it to a normalized device +// coordinates. We divide the NDC range into S1 evenly-sized +// pixels, and assume that each pixel falls in the *center* of its range. +// The default value of the NDC range is [-1, 1], however in the case that +// H != W, the NDC range is set such that the shorter side has range [-1, 1] and +// the longer side is scaled by the ratio of H:W. The dimension of i should be +// S1 and the other image dimension is S2 For example, to get the x and y NDC +// coordinates or a given pixel i: +// x = PixToNonSquareNdc(i, W, H) +// y = PixToNonSquareNdc(i, H, W) +inline float PixToNonSquareNdc(int i, int S1, int S2) { + float range = NonSquareNdcRange(S1, S2); + // NDC: offset + (i * pixel_width + half_pixel_width) + // The NDC range is [-range/2, range/2]. + const float offset = (range / 2.0f); + return -offset + (range * i + offset) / S1; +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_points/rasterize_points.cu b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_points/rasterize_points.cu new file mode 100644 index 0000000000000000000000000000000000000000..5b18d8334d3358244409a2d9e43acd3c9262cb99 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_points/rasterize_points.cu @@ -0,0 +1,462 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "rasterize_points/rasterization_utils.cuh" + +namespace { +// A little structure for holding details about a pixel. +struct Pix { + float z; // Depth of the reference point. + int32_t idx; // Index of the reference point. + float dist2; // Euclidean distance square to the reference point. +}; + +__device__ inline bool operator<(const Pix& a, const Pix& b) { + return a.z < b.z; +} + +// This function checks if a pixel given by xy location pxy lies within the +// point with index p and batch index n. One of the inputs is a list (q) +// which contains Pixel structs with the indices of the points which intersect +// with this pixel sorted by closest z distance. If the pixel pxy lies in the +// point, the list (q) is updated and re-orderered in place. In addition +// the auxiliary variables q_size, q_max_z and q_max_idx are also modified. +// This code is shared between RasterizePointsNaiveCudaKernel and +// RasterizePointsFineCudaKernel. +template +__device__ void CheckPixelInsidePoint( + const float* points, // (P, 3) + const int p_idx, + int& q_size, + float& q_max_z, + int& q_max_idx, + PointQ& q, + const float* radius, + const float xf, + const float yf, + const int K) { + const float px = points[p_idx * 3 + 0]; + const float py = points[p_idx * 3 + 1]; + const float pz = points[p_idx * 3 + 2]; + const float p_radius = radius[p_idx]; + const float radius2 = p_radius * p_radius; + if (pz < 0) + return; // Don't render points behind the camera + const float dx = xf - px; + const float dy = yf - py; + const float dist2 = dx * dx + dy * dy; + if (dist2 < radius2) { + if (q_size < K) { + // Just insert it + q[q_size] = {pz, p_idx, dist2}; + if (pz > q_max_z) { + q_max_z = pz; + q_max_idx = q_size; + } + q_size++; + } else if (pz < q_max_z) { + // Overwrite the old max, and find the new max + q[q_max_idx] = {pz, p_idx, dist2}; + q_max_z = pz; + for (int i = 0; i < K; i++) { + if (q[i].z > q_max_z) { + q_max_z = q[i].z; + q_max_idx = i; + } + } + } + } +} +} // namespace +// **************************************************************************** +// * NAIVE RASTERIZATION * +// **************************************************************************** + +__global__ void RasterizePointsNaiveCudaKernel( + const float* points, // (P, 3) + const int64_t* cloud_to_packed_first_idx, // (N) + const int64_t* num_points_per_cloud, // (N) + const float* radius, + const int N, + const int H, + const int W, + const int K, + int32_t* point_idxs, // (N, H, W, K) + float* zbuf, // (N, H, W, K) + float* pix_dists) { // (N, H, W, K) + // Simple version: One thread per output pixel + const int num_threads = gridDim.x * blockDim.x; + const int tid = blockDim.x * blockIdx.x + threadIdx.x; + for (int i = tid; i < N * H * W; i += num_threads) { + // Convert linear index to 3D index + const int n = i / (H * W); // Batch index + const int pix_idx = i % (H * W); + + // Reverse ordering of the X and Y axis as the camera coordinates + // assume that +Y is pointing up and +X is pointing left. + const int yi = H - 1 - pix_idx / W; + const int xi = W - 1 - pix_idx % W; + + // screen coordinates to ndc coordinates of pixel. + const float xf = PixToNonSquareNdc(xi, W, H); + const float yf = PixToNonSquareNdc(yi, H, W); + + // For keeping track of the K closest points we want a data structure + // that (1) gives O(1) access to the closest point for easy comparisons, + // and (2) allows insertion of new elements. In the CPU version we use + // std::priority_queue; then (2) is O(log K). We can't use STL + // containers in CUDA; we could roll our own max heap in an array, but + // that would likely have a lot of warp divergence so we do something + // simpler instead: keep the elements in an unsorted array, but keep + // track of the max value and the index of the max value. Then (1) is + // still O(1) time, while (2) is O(K) with a clean loop. Since K <= 8 + // this should be fast enough for our purposes. + // TODO(jcjohns) Abstract this out into a standalone data structure + Pix q[kMaxPointsPerPixel]; + int q_size = 0; + float q_max_z = -1000; + int q_max_idx = -1; + + // Using the batch index of the thread get the start and stop + // indices for the points. + const int64_t point_start_idx = cloud_to_packed_first_idx[n]; + const int64_t point_stop_idx = point_start_idx + num_points_per_cloud[n]; + + for (int p_idx = point_start_idx; p_idx < point_stop_idx; ++p_idx) { + CheckPixelInsidePoint( + points, p_idx, q_size, q_max_z, q_max_idx, q, radius, xf, yf, K); + } + BubbleSort(q, q_size); + int idx = n * H * W * K + pix_idx * K; + for (int k = 0; k < q_size; ++k) { + point_idxs[idx + k] = q[k].idx; + zbuf[idx + k] = q[k].z; + pix_dists[idx + k] = q[k].dist2; + } + } +} + +std::tuple RasterizePointsNaiveCuda( + const at::Tensor& points, // (P. 3) + const at::Tensor& cloud_to_packed_first_idx, // (N) + const at::Tensor& num_points_per_cloud, // (N) + const std::tuple image_size, + const at::Tensor& radius, + const int points_per_pixel) { + // Check inputs are on the same device + at::TensorArg points_t{points, "points", 1}, + cloud_to_packed_first_idx_t{ + cloud_to_packed_first_idx, "cloud_to_packed_first_idx", 2}, + num_points_per_cloud_t{num_points_per_cloud, "num_points_per_cloud", 3}; + at::CheckedFrom c = "RasterizePointsNaiveCuda"; + at::checkAllSameGPU( + c, {points_t, cloud_to_packed_first_idx_t, num_points_per_cloud_t}); + + // Set the device for the kernel launch based on the device of the input + at::cuda::CUDAGuard device_guard(points.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + TORCH_CHECK( + points.ndimension() == 2 && points.size(1) == 3, + "points must have dimensions (num_points, 3)"); + TORCH_CHECK( + num_points_per_cloud.size(0) == cloud_to_packed_first_idx.size(0), + "num_points_per_cloud must have same size first dimension as cloud_to_packed_first_idx"); + + const int N = num_points_per_cloud.size(0); // batch size. + const int H = std::get<0>(image_size); + const int W = std::get<1>(image_size); + const int K = points_per_pixel; + + if (K > kMaxPointsPerPixel) { + std::stringstream ss; + ss << "Must have points_per_pixel <= " << kMaxPointsPerPixel; + AT_ERROR(ss.str()); + } + + auto int_opts = num_points_per_cloud.options().dtype(at::kInt); + auto float_opts = points.options().dtype(at::kFloat); + at::Tensor point_idxs = at::full({N, H, W, K}, -1, int_opts); + at::Tensor zbuf = at::full({N, H, W, K}, -1, float_opts); + at::Tensor pix_dists = at::full({N, H, W, K}, -1, float_opts); + + if (point_idxs.numel() == 0) { + AT_CUDA_CHECK(cudaGetLastError()); + return std::make_tuple(point_idxs, zbuf, pix_dists); + } + + const size_t blocks = 1024; + const size_t threads = 64; + RasterizePointsNaiveCudaKernel<<>>( + points.contiguous().data_ptr(), + cloud_to_packed_first_idx.contiguous().data_ptr(), + num_points_per_cloud.contiguous().data_ptr(), + radius.contiguous().data_ptr(), + N, + H, + W, + K, + point_idxs.contiguous().data_ptr(), + zbuf.contiguous().data_ptr(), + pix_dists.contiguous().data_ptr()); + + AT_CUDA_CHECK(cudaGetLastError()); + return std::make_tuple(point_idxs, zbuf, pix_dists); +} + +// **************************************************************************** +// * FINE RASTERIZATION * +// **************************************************************************** + +__global__ void RasterizePointsFineCudaKernel( + const float* points, // (P, 3) + const int32_t* bin_points, // (N, BH, BW, T) + const float* radius, + const int bin_size, + const int N, + const int BH, // num_bins y + const int BW, // num_bins x + const int M, + const int H, + const int W, + const int K, + int32_t* point_idxs, // (N, H, W, K) + float* zbuf, // (N, H, W, K) + float* pix_dists) { // (N, H, W, K) + // This can be more than H * W if H or W are not divisible by bin_size. + const int num_pixels = N * BH * BW * bin_size * bin_size; + const int num_threads = gridDim.x * blockDim.x; + const int tid = blockIdx.x * blockDim.x + threadIdx.x; + + for (int pid = tid; pid < num_pixels; pid += num_threads) { + // Convert linear index into bin and pixel indices. We make the within + // block pixel ids move the fastest, so that adjacent threads will fall + // into the same bin; this should give them coalesced memory reads when + // they read from points and bin_points. + int i = pid; + const int n = i / (BH * BW * bin_size * bin_size); + i %= BH * BW * bin_size * bin_size; + const int by = i / (BW * bin_size * bin_size); + i %= BW * bin_size * bin_size; + const int bx = i / (bin_size * bin_size); + i %= bin_size * bin_size; + + const int yi = i / bin_size + by * bin_size; + const int xi = i % bin_size + bx * bin_size; + + if (yi >= H || xi >= W) + continue; + + const float xf = PixToNonSquareNdc(xi, W, H); + const float yf = PixToNonSquareNdc(yi, H, W); + + // This part looks like the naive rasterization kernel, except we use + // bin_points to only look at a subset of points already known to fall + // in this bin. TODO abstract out this logic into some data structure + // that is shared by both kernels? + Pix q[kMaxPointsPerPixel]; + int q_size = 0; + float q_max_z = -1000; + int q_max_idx = -1; + for (int m = 0; m < M; ++m) { + const int p = bin_points[n * BH * BW * M + by * BW * M + bx * M + m]; + if (p < 0) { + // bin_points uses -1 as a sentinal value + continue; + } + CheckPixelInsidePoint( + points, p, q_size, q_max_z, q_max_idx, q, radius, xf, yf, K); + } + // Now we've looked at all the points for this bin, so we can write + // output for the current pixel. + BubbleSort(q, q_size); + + // Reverse ordering of the X and Y axis as the camera coordinates + // assume that +Y is pointing up and +X is pointing left. + const int yidx = H - 1 - yi; + const int xidx = W - 1 - xi; + + const int pix_idx = n * H * W * K + yidx * W * K + xidx * K; + for (int k = 0; k < q_size; ++k) { + point_idxs[pix_idx + k] = q[k].idx; + zbuf[pix_idx + k] = q[k].z; + pix_dists[pix_idx + k] = q[k].dist2; + } + } +} + +std::tuple RasterizePointsFineCuda( + const at::Tensor& points, // (P, 3) + const at::Tensor& bin_points, + const std::tuple image_size, + const at::Tensor& radius, + const int bin_size, + const int points_per_pixel) { + // Check inputs are on the same device + at::TensorArg points_t{points, "points", 1}, + bin_points_t{bin_points, "bin_points", 2}; + at::CheckedFrom c = "RasterizePointsFineCuda"; + at::checkAllSameGPU(c, {points_t, bin_points_t}); + + // Set the device for the kernel launch based on the device of the input + at::cuda::CUDAGuard device_guard(points.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + const int N = bin_points.size(0); + const int BH = bin_points.size(1); + const int BW = bin_points.size(2); + const int M = bin_points.size(3); + const int K = points_per_pixel; + + const int H = std::get<0>(image_size); + const int W = std::get<1>(image_size); + + if (K > kMaxPointsPerPixel) { + AT_ERROR("Must have num_closest <= 150"); + } + auto int_opts = bin_points.options().dtype(at::kInt); + auto float_opts = points.options().dtype(at::kFloat); + at::Tensor point_idxs = at::full({N, H, W, K}, -1, int_opts); + at::Tensor zbuf = at::full({N, H, W, K}, -1, float_opts); + at::Tensor pix_dists = at::full({N, H, W, K}, -1, float_opts); + + if (point_idxs.numel() == 0) { + AT_CUDA_CHECK(cudaGetLastError()); + return std::make_tuple(point_idxs, zbuf, pix_dists); + } + + const size_t blocks = 1024; + const size_t threads = 64; + RasterizePointsFineCudaKernel<<>>( + points.contiguous().data_ptr(), + bin_points.contiguous().data_ptr(), + radius.contiguous().data_ptr(), + bin_size, + N, + BH, + BW, + M, + H, + W, + K, + point_idxs.contiguous().data_ptr(), + zbuf.contiguous().data_ptr(), + pix_dists.contiguous().data_ptr()); + + AT_CUDA_CHECK(cudaGetLastError()); + return std::make_tuple(point_idxs, zbuf, pix_dists); +} + +// **************************************************************************** +// * BACKWARD PASS * +// **************************************************************************** +// TODO(T55115174) Add more documentation for backward kernel. +__global__ void RasterizePointsBackwardCudaKernel( + const float* points, // (P, 3) + const int32_t* idxs, // (N, H, W, K) + const int N, + const int P, + const int H, + const int W, + const int K, + const float* grad_zbuf, // (N, H, W, K) + const float* grad_dists, // (N, H, W, K) + float* grad_points) { // (P, 3) + // Parallelized over each of K points per pixel, for each pixel in images of + // size H * W, for each image in the batch of size N. + int num_threads = gridDim.x * blockDim.x; + int tid = blockIdx.x * blockDim.x + threadIdx.x; + for (int i = tid; i < N * H * W * K; i += num_threads) { + // const int n = i / (H * W * K); // batch index (not needed). + const int yxk = i % (H * W * K); + const int yi = yxk / (W * K); + const int xk = yxk % (W * K); + const int xi = xk / K; + // k = xk % K (We don't actually need k, but this would be it.) + // Reverse ordering of X and Y axes. + const int yidx = H - 1 - yi; + const int xidx = W - 1 - xi; + + const float xf = PixToNonSquareNdc(xidx, W, H); + const float yf = PixToNonSquareNdc(yidx, H, W); + + const int p = idxs[i]; + if (p < 0) + continue; + const float grad_dist2 = grad_dists[i]; + const int p_ind = p * 3; // index into packed points tensor + const float px = points[p_ind + 0]; + const float py = points[p_ind + 1]; + const float dx = px - xf; + const float dy = py - yf; + const float grad_px = 2.0f * grad_dist2 * dx; + const float grad_py = 2.0f * grad_dist2 * dy; + const float grad_pz = grad_zbuf[i]; + atomicAdd(grad_points + p_ind + 0, grad_px); + atomicAdd(grad_points + p_ind + 1, grad_py); + atomicAdd(grad_points + p_ind + 2, grad_pz); + } +} + +at::Tensor RasterizePointsBackwardCuda( + const at::Tensor& points, // (N, P, 3) + const at::Tensor& idxs, // (N, H, W, K) + const at::Tensor& grad_zbuf, // (N, H, W, K) + const at::Tensor& grad_dists) { // (N, H, W, K) + + // Check inputs are on the same device + at::TensorArg points_t{points, "points", 1}, idxs_t{idxs, "idxs", 2}, + grad_zbuf_t{grad_zbuf, "grad_zbuf", 3}, + grad_dists_t{grad_dists, "grad_dists", 4}; + at::CheckedFrom c = "RasterizePointsBackwardCuda"; + at::checkAllSameGPU(c, {points_t, idxs_t, grad_zbuf_t, grad_dists_t}); + at::checkAllSameType(c, {points_t, grad_zbuf_t, grad_dists_t}); + // This is nondeterministic because atomicAdd + at::globalContext().alertNotDeterministic("RasterizePointsBackwardCuda"); + // Set the device for the kernel launch based on the device of the input + at::cuda::CUDAGuard device_guard(points.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + const int P = points.size(0); + const int N = idxs.size(0); + const int H = idxs.size(1); + const int W = idxs.size(2); + const int K = idxs.size(3); + + at::Tensor grad_points = at::zeros({P, 3}, points.options()); + + if (grad_points.numel() == 0) { + AT_CUDA_CHECK(cudaGetLastError()); + return grad_points; + } + + const size_t blocks = 1024; + const size_t threads = 64; + + RasterizePointsBackwardCudaKernel<<>>( + points.contiguous().data_ptr(), + idxs.contiguous().data_ptr(), + N, + P, + H, + W, + K, + grad_zbuf.contiguous().data_ptr(), + grad_dists.contiguous().data_ptr(), + grad_points.contiguous().data_ptr()); + + AT_CUDA_CHECK(cudaGetLastError()); + return grad_points; +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_points/rasterize_points.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_points/rasterize_points.h new file mode 100644 index 0000000000000000000000000000000000000000..4e74e9e2b0600dccbcbe3e745e80f13a8149ca99 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_points/rasterize_points.h @@ -0,0 +1,360 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once +#include +#include +#include +#include "rasterize_coarse/rasterize_coarse.h" +#include "utils/pytorch3d_cutils.h" + +// **************************************************************************** +// * NAIVE RASTERIZATION * +// **************************************************************************** + +std::tuple RasterizePointsNaiveCpu( + const torch::Tensor& points, + const torch::Tensor& cloud_to_packed_first_idx, + const torch::Tensor& num_points_per_cloud, + const std::tuple image_size, + const torch::Tensor& radius, + const int points_per_pixel); + +#ifdef WITH_CUDA +std::tuple +RasterizePointsNaiveCuda( + const torch::Tensor& points, + const torch::Tensor& cloud_to_packed_first_idx, + const torch::Tensor& num_points_per_cloud, + const std::tuple image_size, + const torch::Tensor& radius, + const int points_per_pixel); +#endif +// Naive (forward) pointcloud rasterization: For each pixel, for each point, +// check whether that point hits the pixel. +// +// Args: +// points: Tensor of shape (P, 3) giving (packed) positions for +// points in all N pointclouds in the batch where P is the total +// number of points in the batch across all pointclouds. These points +// are expected to be in NDC coordinates in the range [-1, 1]. +// cloud_to_packed_first_idx: LongTensor of shape (N) giving the index in +// points_packed of the first point in each pointcloud +// in the batch where N is the batch size. +// num_points_per_cloud: LongTensor of shape (N) giving the number of points +// for each pointcloud in the batch. +// image_size: Tuple (H, W) giving the size in pixels of the output +// image to be rasterized. +// radius: FloatTensor of shape (P) giving the radius (in NDC units) of +// each point in points. +// points_per_pixel: (K) The number closest of points to return for each pixel +// +// Returns: +// A 4 element tuple of: +// idxs: int32 Tensor of shape (N, S, S, K) giving the indices of the +// closest K points along the z-axis for each pixel, padded with -1 for +// pixels hit by fewer than K points. The indices refer to points in +// points packed i.e a tensor of shape (P, 3) representing the flattened +// points for all pointclouds in the batch. +// zbuf: float32 Tensor of shape (N, S, S, K) giving the depth of each +// closest point for each pixel. +// dists: float32 Tensor of shape (N, S, S, K) giving squared Euclidean +// distance in the (NDC) x/y plane between each pixel and its K closest +// points along the z axis. +std::tuple RasterizePointsNaive( + const torch::Tensor& points, + const torch::Tensor& cloud_to_packed_first_idx, + const torch::Tensor& num_points_per_cloud, + const std::tuple image_size, + const torch::Tensor& radius, + const int points_per_pixel) { + if (points.is_cuda() && cloud_to_packed_first_idx.is_cuda() && + num_points_per_cloud.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(points); + CHECK_CUDA(cloud_to_packed_first_idx); + CHECK_CUDA(num_points_per_cloud); + CHECK_CUDA(radius); + return RasterizePointsNaiveCuda( + points, + cloud_to_packed_first_idx, + num_points_per_cloud, + image_size, + radius, + points_per_pixel); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } else { + return RasterizePointsNaiveCpu( + points, + cloud_to_packed_first_idx, + num_points_per_cloud, + image_size, + radius, + points_per_pixel); + } +} + +// **************************************************************************** +// * COARSE RASTERIZATION * +// **************************************************************************** + +// RasterizePointsCoarseCuda in rasterize_coarse/rasterize_coarse.h + +torch::Tensor RasterizePointsCoarseCpu( + const torch::Tensor& points, + const torch::Tensor& cloud_to_packed_first_idx, + const torch::Tensor& num_points_per_cloud, + const std::tuple image_size, + const torch::Tensor& radius, + const int bin_size, + const int max_points_per_bin); + +// Args: +// points: Tensor of shape (P, 3) giving (packed) positions for +// points in all N pointclouds in the batch where P is the total +// number of points in the batch across all pointclouds. These points +// are expected to be in NDC coordinates in the range [-1, 1]. +// cloud_to_packed_first_idx: LongTensor of shape (N) giving the index in +// points_packed of the first point in each pointcloud +// in the batch where N is the batch size. +// num_points_per_cloud: LongTensor of shape (N) giving the number of points +// for each pointcloud in the batch. +// image_size: Tuple (H, W) giving the size in pixels of the output +// image to be rasterized. +// radius: FloatTensor of shape (P) giving the radius (in NDC units) of +// each point in points. +// bin_size: Size of each bin within the image (in pixels) +// max_points_per_bin: The maximum number of points allowed to fall into each +// bin when using coarse-to-fine rasterization. +// +// Returns: +// points_per_bin: Tensor of shape (N, num_bins, num_bins) giving the number +// of points that fall in each bin +// bin_points: Tensor of shape (N, num_bins, num_bins, K) giving the indices +// of points that fall into each bin. +torch::Tensor RasterizePointsCoarse( + const torch::Tensor& points, + const torch::Tensor& cloud_to_packed_first_idx, + const torch::Tensor& num_points_per_cloud, + const std::tuple image_size, + const torch::Tensor& radius, + const int bin_size, + const int max_points_per_bin) { + if (points.is_cuda() && cloud_to_packed_first_idx.is_cuda() && + num_points_per_cloud.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(points); + CHECK_CUDA(cloud_to_packed_first_idx); + CHECK_CUDA(num_points_per_cloud); + CHECK_CUDA(radius); + return RasterizePointsCoarseCuda( + points, + cloud_to_packed_first_idx, + num_points_per_cloud, + image_size, + radius, + bin_size, + max_points_per_bin); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } else { + return RasterizePointsCoarseCpu( + points, + cloud_to_packed_first_idx, + num_points_per_cloud, + image_size, + radius, + bin_size, + max_points_per_bin); + } +} + +// **************************************************************************** +// * FINE RASTERIZATION * +// **************************************************************************** + +#ifdef WITH_CUDA +std::tuple RasterizePointsFineCuda( + const torch::Tensor& points, + const torch::Tensor& bin_points, + const std::tuple image_size, + const torch::Tensor& radius, + const int bin_size, + const int points_per_pixel); +#endif +// Args: +// points: Tensor of shape (P, 3) giving (packed) positions for +// points in all N pointclouds in the batch where P is the total +// number of points in the batch across all pointclouds. These points +// are expected to be in NDC coordinates in the range [-1, 1]. +// bin_points: int32 Tensor of shape (N, B, B, M) giving the indices of points +// that fall into each bin (output from coarse rasterization) +// image_size: Tuple (H, W) giving the size in pixels of the output +// image to be rasterized. +// radius: FloatTensor of shape (P) giving the radius (in NDC units) of +// each point in points. +// bin_size: Size of each bin (in pixels) +// points_per_pixel: How many points to rasterize for each pixel +// +// Returns (same as rasterize_points): +// idxs: int32 Tensor of shape (N, S, S, K) giving the indices of the +// closest K points along the z-axis for each pixel, padded with -1 for +// pixels hit by fewer than K points. The indices refer to points in +// points packed i.e a tensor of shape (P, 3) representing the flattened +// points for all pointclouds in the batch. +// zbuf: float32 Tensor of shape (N, S, S, K) giving the depth of each of each +// closest point for each pixel +// dists: float32 Tensor of shape (N, S, S, K) giving squared Euclidean +// distance in the (NDC) x/y plane between each pixel and its K closest +// points along the z axis. +std::tuple RasterizePointsFine( + const torch::Tensor& points, + const torch::Tensor& bin_points, + const std::tuple image_size, + const torch::Tensor& radius, + const int bin_size, + const int points_per_pixel) { + if (points.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(points); + CHECK_CUDA(bin_points); + return RasterizePointsFineCuda( + points, bin_points, image_size, radius, bin_size, points_per_pixel); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } else { + AT_ERROR("NOT IMPLEMENTED"); + } +} + +// **************************************************************************** +// * BACKWARD PASS * +// **************************************************************************** + +torch::Tensor RasterizePointsBackwardCpu( + const torch::Tensor& points, + const torch::Tensor& idxs, + const torch::Tensor& grad_zbuf, + const torch::Tensor& grad_dists); + +#ifdef WITH_CUDA +torch::Tensor RasterizePointsBackwardCuda( + const torch::Tensor& points, + const torch::Tensor& idxs, + const torch::Tensor& grad_zbuf, + const torch::Tensor& grad_dists); +#endif +// Args: +// points: Tensor of shape (P, 3) giving (packed) positions for +// points in all N pointclouds in the batch where P is the total +// number of points in the batch across all pointclouds. These points +// are expected to be in NDC coordinates in the range [-1, 1]. +// idxs: int32 Tensor of shape (N, H, W, K) (from forward pass) +// grad_zbuf: float32 Tensor of shape (N, H, W, K) giving upstream gradient +// d(loss)/d(zbuf) of the distances from each pixel to its nearest +// points. +// grad_dists: Tensor of shape (N, H, W, K) giving upstream gradient +// d(loss)/d(dists) of the dists tensor returned by the forward +// pass. +// +// Returns: +// grad_points: float32 Tensor of shape (N, P, 3) giving downstream gradients +torch::Tensor RasterizePointsBackward( + const torch::Tensor& points, + const torch::Tensor& idxs, + const torch::Tensor& grad_zbuf, + const torch::Tensor& grad_dists) { + if (points.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(points); + CHECK_CUDA(idxs); + CHECK_CUDA(grad_zbuf); + CHECK_CUDA(grad_dists); + return RasterizePointsBackwardCuda(points, idxs, grad_zbuf, grad_dists); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } else { + return RasterizePointsBackwardCpu(points, idxs, grad_zbuf, grad_dists); + } +} + +// **************************************************************************** +// * MAIN ENTRY POINT * +// **************************************************************************** + +// This is the main entry point for the forward pass of the point rasterizer; +// it uses either naive or coarse-to-fine rasterization based on bin_size. +// +// Args: +// points: Tensor of shape (P, 3) giving (packed) positions for +// points in all N pointclouds in the batch where P is the total +// number of points in the batch across all pointclouds. These points +// are expected to be in NDC coordinates in the range [-1, 1]. +// cloud_to_packed_first_idx: LongTensor of shape (N) giving the index in +// points_packed of the first point in each pointcloud +// in the batch where N is the batch size. +// num_points_per_cloud: LongTensor of shape (N) giving the number of points +// for each pointcloud in the batch. +// image_size: Tuple (H, W) giving the size in pixels of the output +// image to be rasterized. +// radius: FloatTensor of shape (P) giving the radius (in NDC units) of +// each point in points. +// points_per_pixel: (K) The number of points to return for each pixel +// bin_size: Bin size (in pixels) for coarse-to-fine rasterization. Setting +// bin_size=0 uses naive rasterization instead. +// max_points_per_bin: The maximum number of points allowed to fall into each +// bin when using coarse-to-fine rasterization. +// +// Returns: +// idxs: int32 Tensor of shape (N, S, S, K) giving the indices of the +// closest K points along the z-axis for each pixel, padded with -1 for +// pixels hit by fewer than K points. The indices refer to points in +// points packed i.e a tensor of shape (P, 3) representing the flattened +// points for all pointclouds in the batch. +// zbuf: float32 Tensor of shape (N, S, S, K) giving the depth of each of each +// closest point for each pixel +// dists: float32 Tensor of shape (N, S, S, K) giving squared Euclidean +// distance in the (NDC) x/y plane between each pixel and its K closest +// points along the z axis. +std::tuple RasterizePoints( + const torch::Tensor& points, + const torch::Tensor& cloud_to_packed_first_idx, + const torch::Tensor& num_points_per_cloud, + const std::tuple image_size, + const torch::Tensor& radius, + const int points_per_pixel, + const int bin_size, + const int max_points_per_bin) { + if (bin_size == 0) { + // Use the naive per-pixel implementation + return RasterizePointsNaive( + points, + cloud_to_packed_first_idx, + num_points_per_cloud, + image_size, + radius, + points_per_pixel); + } else { + // Use coarse-to-fine rasterization + const auto bin_points = RasterizePointsCoarse( + points, + cloud_to_packed_first_idx, + num_points_per_cloud, + image_size, + radius, + bin_size, + max_points_per_bin); + return RasterizePointsFine( + points, bin_points, image_size, radius, bin_size, points_per_pixel); + } +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_points/rasterize_points_cpu.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_points/rasterize_points_cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..29da4c497b89b2fe3e2e2d5b2563adbbcd6ef326 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/rasterize_points/rasterize_points_cpu.cpp @@ -0,0 +1,251 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include +#include "rasterization_utils.h" + +std::tuple RasterizePointsNaiveCpu( + const torch::Tensor& points, // (P, 3) + const torch::Tensor& cloud_to_packed_first_idx, // (N) + const torch::Tensor& num_points_per_cloud, // (N) + const std::tuple image_size, + const torch::Tensor& radius, + const int points_per_pixel) { + const int32_t N = cloud_to_packed_first_idx.size(0); // batch_size. + + const int H = std::get<0>(image_size); + const int W = std::get<1>(image_size); + const int K = points_per_pixel; + + // Initialize output tensors. + auto int_opts = num_points_per_cloud.options().dtype(torch::kInt32); + auto float_opts = points.options().dtype(torch::kFloat32); + torch::Tensor point_idxs = torch::full({N, H, W, K}, -1, int_opts); + torch::Tensor zbuf = torch::full({N, H, W, K}, -1, float_opts); + torch::Tensor pix_dists = torch::full({N, H, W, K}, -1, float_opts); + + auto points_a = points.accessor(); + auto point_idxs_a = point_idxs.accessor(); + auto zbuf_a = zbuf.accessor(); + auto pix_dists_a = pix_dists.accessor(); + auto radius_a = radius.accessor(); + + for (int n = 0; n < N; ++n) { + // Loop through each pointcloud in the batch. + // Get the start index of the points in points_packed and the num points + // in the point cloud. + const int point_start_idx = + cloud_to_packed_first_idx[n].item().to(); + const int point_stop_idx = + (point_start_idx + num_points_per_cloud[n].item().to()); + + for (int yi = 0; yi < H; ++yi) { + // Reverse the order of yi so that +Y is pointing upwards in the image. + const int yidx = H - 1 - yi; + const float yf = PixToNonSquareNdc(yidx, H, W); + + for (int xi = 0; xi < W; ++xi) { + // Reverse the order of xi so that +X is pointing to the left in the + // image. + const int xidx = W - 1 - xi; + const float xf = PixToNonSquareNdc(xidx, W, H); + + // Use a priority queue to hold (z, idx, r) + std::priority_queue> q; + for (int p = point_start_idx; p < point_stop_idx; ++p) { + const float px = points_a[p][0]; + const float py = points_a[p][1]; + const float pz = points_a[p][2]; + const float p_radius = radius_a[p]; + const float radius2 = p_radius * p_radius; + if (pz < 0) { + continue; + } + const float dx = px - xf; + const float dy = py - yf; + const float dist2 = dx * dx + dy * dy; + if (dist2 < radius2) { + // The current point hit the current pixel + q.emplace(pz, p, dist2); + if ((int)q.size() > K) { + q.pop(); + } + } + } + // Now all the points have been seen, so pop elements off the queue + // one by one and write them into the output tensors. + while (!q.empty()) { + auto t = q.top(); + q.pop(); + int i = q.size(); + zbuf_a[n][yi][xi][i] = std::get<0>(t); + point_idxs_a[n][yi][xi][i] = std::get<1>(t); + pix_dists_a[n][yi][xi][i] = std::get<2>(t); + } + } + } + } + return std::make_tuple(point_idxs, zbuf, pix_dists); +} + +torch::Tensor RasterizePointsCoarseCpu( + const torch::Tensor& points, // (P, 3) + const torch::Tensor& cloud_to_packed_first_idx, // (N) + const torch::Tensor& num_points_per_cloud, // (N) + const std::tuple image_size, + const torch::Tensor& radius, + const int bin_size, + const int max_points_per_bin) { + const int32_t N = cloud_to_packed_first_idx.size(0); // batch_size. + const int M = max_points_per_bin; + + const float H = std::get<0>(image_size); + const float W = std::get<1>(image_size); + + // Integer division round up. + const int BH = 1 + (H - 1) / bin_size; + const int BW = 1 + (W - 1) / bin_size; + + auto opts = num_points_per_cloud.options().dtype(torch::kInt32); + torch::Tensor points_per_bin = torch::zeros({N, BH, BW}, opts); + torch::Tensor bin_points = torch::full({N, BH, BW, M}, -1, opts); + + auto points_a = points.accessor(); + auto points_per_bin_a = points_per_bin.accessor(); + auto bin_points_a = bin_points.accessor(); + auto radius_a = radius.accessor(); + + const float ndc_x_range = NonSquareNdcRange(W, H); + const float pixel_width_x = ndc_x_range / W; + const float bin_width_x = pixel_width_x * bin_size; + + const float ndc_y_range = NonSquareNdcRange(H, W); + const float pixel_width_y = ndc_y_range / H; + const float bin_width_y = pixel_width_y * bin_size; + + for (int n = 0; n < N; ++n) { + // Loop through each pointcloud in the batch. + // Get the start index of the points in points_packed and the num points + // in the point cloud. + const int point_start_idx = + cloud_to_packed_first_idx[n].item().to(); + const int point_stop_idx = + (point_start_idx + num_points_per_cloud[n].item().to()); + + float bin_y_min = -1.0f; + float bin_y_max = bin_y_min + bin_width_y; + + // Iterate through the horizontal bins from top to bottom. + for (int by = 0; by < BH; by++) { + float bin_x_min = -1.0f; + float bin_x_max = bin_x_min + bin_width_x; + + // Iterate through bins on this horizontal line, left to right. + for (int bx = 0; bx < BW; bx++) { + int32_t points_hit = 0; + for (int p = point_start_idx; p < point_stop_idx; ++p) { + float px = points_a[p][0]; + float py = points_a[p][1]; + float pz = points_a[p][2]; + const float p_radius = radius_a[p]; + if (pz < 0) { + continue; + } + float point_x_min = px - p_radius; + float point_x_max = px + p_radius; + float point_y_min = py - p_radius; + float point_y_max = py + p_radius; + + // Use a half-open interval so that points exactly on the + // boundary between bins will fall into exactly one bin. + bool x_hit = (point_x_min <= bin_x_max) && (bin_x_min <= point_x_max); + bool y_hit = (point_y_min <= bin_y_max) && (bin_y_min <= point_y_max); + if (x_hit && y_hit) { + // Got too many points for this bin, so throw an error. + if (points_hit >= max_points_per_bin) { + AT_ERROR("Got too many points per bin"); + } + // The current point falls in the current bin, so + // record it. + bin_points_a[n][by][bx][points_hit] = p; + points_hit++; + } + } + // Record the number of points found in this bin + points_per_bin_a[n][by][bx] = points_hit; + + // Shift the bin to the right for the next loop iteration + bin_x_min = bin_x_max; + bin_x_max = bin_x_min + bin_width_x; + } + // Shift the bin down for the next loop iteration + bin_y_min = bin_y_max; + bin_y_max = bin_y_min + bin_width_y; + } + } + return bin_points; +} + +torch::Tensor RasterizePointsBackwardCpu( + const torch::Tensor& points, // (P, 3) + const torch::Tensor& idxs, // (N, H, W, K) + const torch::Tensor& grad_zbuf, // (N, H, W, K) + const torch::Tensor& grad_dists) { // (N, H, W, K) + + const int N = idxs.size(0); + const int P = points.size(0); + const int H = idxs.size(1); + const int W = idxs.size(2); + const int K = idxs.size(3); + + torch::Tensor grad_points = torch::zeros({P, 3}, points.options()); + + auto points_a = points.accessor(); + auto idxs_a = idxs.accessor(); + auto grad_dists_a = grad_dists.accessor(); + auto grad_zbuf_a = grad_zbuf.accessor(); + auto grad_points_a = grad_points.accessor(); + + for (int n = 0; n < N; ++n) { // Loop over images in the batch + for (int y = 0; y < H; ++y) { // Loop over rows in the image + // Reverse the order of yi so that +Y is pointing upwards in the image. + const int yidx = H - 1 - y; + // Y coordinate of the top of the pixel. + const float yf = PixToNonSquareNdc(yidx, H, W); + + // Iterate through pixels on this horizontal line, left to right. + for (int x = 0; x < W; ++x) { // Loop over pixels in the row + + // Reverse the order of xi so that +X is pointing to the left in the + // image. + const int xidx = W - 1 - x; + const float xf = PixToNonSquareNdc(xidx, W, H); + for (int k = 0; k < K; ++k) { // Loop over points for the pixel + const int p = idxs_a[n][y][x][k]; + if (p < 0) { + break; + } + const float grad_dist2 = grad_dists_a[n][y][x][k]; + const float px = points_a[p][0]; + const float py = points_a[p][1]; + const float dx = px - xf; + const float dy = py - yf; + // Remember: dists[n][y][x][k] = dx * dx + dy * dy; + const float grad_px = 2.0f * grad_dist2 * dx; + const float grad_py = 2.0f * grad_dist2 * dy; + grad_points_a[p][0] += grad_px; + grad_points_a[p][1] += grad_py; + grad_points_a[p][2] += grad_zbuf_a[n][y][x][k]; + } + } + } + } + return grad_points; +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/sample_farthest_points/sample_farthest_points.cu b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/sample_farthest_points/sample_farthest_points.cu new file mode 100644 index 0000000000000000000000000000000000000000..70cef75c7b4bd57cdf4615f415786bbefc288a57 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/sample_farthest_points/sample_farthest_points.cu @@ -0,0 +1,226 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "utils/warp_reduce.cuh" + +template +__global__ void FarthestPointSamplingKernel( + // clang-format off + const at::PackedTensorAccessor64 points, + const at::PackedTensorAccessor64 lengths, + const at::PackedTensorAccessor64 K, + at::PackedTensorAccessor64 idxs, + at::PackedTensorAccessor64 min_point_dist, + const at::PackedTensorAccessor64 start_idxs + // clang-format on +) { + typedef cub::BlockReduce< + cub::KeyValuePair, + block_size, + cub::BLOCK_REDUCE_RAKING_COMMUTATIVE_ONLY> + BlockReduce; + __shared__ typename BlockReduce::TempStorage temp_storage; + __shared__ int64_t selected_store; + + // Get constants + const int64_t N = points.size(0); + const int64_t P = points.size(1); + const int64_t D = points.size(2); + + // Get batch index and thread index + const int64_t batch_idx = blockIdx.x; + const size_t tid = threadIdx.x; + + // If K is greater than the number of points in the pointcloud + // we only need to iterate until the smaller value is reached. + const int64_t k_n = min(K[batch_idx], lengths[batch_idx]); + + // Write the first selected point to global memory in the first thread + int64_t selected = start_idxs[batch_idx]; + if (tid == 0) + idxs[batch_idx][0] = selected; + + // Iterate to find k_n sampled points + for (int64_t k = 1; k < k_n; ++k) { + // Keep track of the maximum of the minimum distance to previously selected + // points seen by this thread + int64_t max_dist_idx = 0; + float max_dist = -1.0; + + // Iterate through all the points in this pointcloud. For already selected + // points, the minimum distance to the set of previously selected points + // will be 0.0 so they won't be selected again. + for (int64_t p = tid; p < lengths[batch_idx]; p += block_size) { + // Calculate the distance to the last selected point + float dist2 = 0.0; + for (int64_t d = 0; d < D; ++d) { + float diff = points[batch_idx][selected][d] - points[batch_idx][p][d]; + dist2 += (diff * diff); + } + + // If the distance of point p to the last selected point is + // less than the previous minimum distance of p to the set of selected + // points, then updated the corresponding value in min_point_dist + // so it always contains the min distance. + const float p_min_dist = min(dist2, min_point_dist[batch_idx][p]); + min_point_dist[batch_idx][p] = p_min_dist; + + // Update the max distance and point idx for this thread. + max_dist_idx = (p_min_dist > max_dist) ? p : max_dist_idx; + max_dist = (p_min_dist > max_dist) ? p_min_dist : max_dist; + } + + // max_dist, max_dist_idx are now the max point and idx seen by this thread. + // Now find the index corresponding to the maximum distance seen by any + // thread. (This value is only on thread 0.) + selected = + BlockReduce(temp_storage) + .Reduce( + cub::KeyValuePair(max_dist_idx, max_dist), + cub::ArgMax(), + block_size) + .key; + + if (tid == 0) { + // Write the farthest point for iteration k to global memory + idxs[batch_idx][k] = selected; + selected_store = selected; + } + + // Ensure `selected` in all threads equals the global maximum. + __syncthreads(); + selected = selected_store; + } +} + +at::Tensor FarthestPointSamplingCuda( + const at::Tensor& points, // (N, P, 3) + const at::Tensor& lengths, // (N,) + const at::Tensor& K, // (N,) + const at::Tensor& start_idxs) { + // Check inputs are on the same device + at::TensorArg p_t{points, "points", 1}, lengths_t{lengths, "lengths", 2}, + k_t{K, "K", 3}, start_idxs_t{start_idxs, "start_idxs", 4}; + at::CheckedFrom c = "FarthestPointSamplingCuda"; + at::checkAllSameGPU(c, {p_t, lengths_t, k_t, start_idxs_t}); + at::checkAllSameType(c, {lengths_t, k_t, start_idxs_t}); + + // Set the device for the kernel launch based on the device of points + at::cuda::CUDAGuard device_guard(points.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + TORCH_CHECK( + points.size(0) == lengths.size(0), + "Point and lengths must have the same batch dimension"); + + TORCH_CHECK( + points.size(0) == K.size(0), + "Points and K must have the same batch dimension"); + + const int64_t N = points.size(0); + const int64_t P = points.size(1); + const int64_t max_K = at::max(K).item(); + + // Initialize the output tensor with the sampled indices + auto idxs = at::full({N, max_K}, -1, lengths.options()); + auto min_point_dist = at::full({N, P}, 1e10, points.options()); + + if (N == 0 || P == 0) { + AT_CUDA_CHECK(cudaGetLastError()); + return idxs; + } + + // Set the number of blocks to the batch size so that the + // block reduction step can be done for each pointcloud + // to find the max distance point in the pointcloud at each iteration. + const size_t blocks = N; + + // Set the threads to the nearest power of 2 of the number of + // points in the pointcloud (up to the max threads in a block). + // This will ensure each thread processes the minimum necessary number of + // points (P/threads). + const int points_pow_2 = std::log(static_cast(P)) / std::log(2.0); + + // Max possible threads per block + const int MAX_THREADS_PER_BLOCK = 1024; + const size_t threads = max(min(1 << points_pow_2, MAX_THREADS_PER_BLOCK), 2); + + // Create the accessors + auto points_a = points.packed_accessor64(); + auto lengths_a = + lengths.packed_accessor64(); + auto K_a = K.packed_accessor64(); + auto idxs_a = idxs.packed_accessor64(); + auto start_idxs_a = + start_idxs.packed_accessor64(); + auto min_point_dist_a = + min_point_dist.packed_accessor64(); + + // TempStorage for the reduction uses static shared memory only. + size_t shared_mem = 0; + + // Support a case for all powers of 2 up to MAX_THREADS_PER_BLOCK possible per + // block. + switch (threads) { + case 1024: + FarthestPointSamplingKernel<1024> + <<>>( + points_a, lengths_a, K_a, idxs_a, min_point_dist_a, start_idxs_a); + break; + case 512: + FarthestPointSamplingKernel<512><<>>( + points_a, lengths_a, K_a, idxs_a, min_point_dist_a, start_idxs_a); + break; + case 256: + FarthestPointSamplingKernel<256><<>>( + points_a, lengths_a, K_a, idxs_a, min_point_dist_a, start_idxs_a); + break; + case 128: + FarthestPointSamplingKernel<128><<>>( + points_a, lengths_a, K_a, idxs_a, min_point_dist_a, start_idxs_a); + break; + case 64: + FarthestPointSamplingKernel<64><<>>( + points_a, lengths_a, K_a, idxs_a, min_point_dist_a, start_idxs_a); + break; + case 32: + FarthestPointSamplingKernel<32><<>>( + points_a, lengths_a, K_a, idxs_a, min_point_dist_a, start_idxs_a); + break; + case 16: + FarthestPointSamplingKernel<16><<>>( + points_a, lengths_a, K_a, idxs_a, min_point_dist_a, start_idxs_a); + break; + case 8: + FarthestPointSamplingKernel<8><<>>( + points_a, lengths_a, K_a, idxs_a, min_point_dist_a, start_idxs_a); + break; + case 4: + FarthestPointSamplingKernel<4><<>>( + points_a, lengths_a, K_a, idxs_a, min_point_dist_a, start_idxs_a); + break; + case 2: + FarthestPointSamplingKernel<2><<>>( + points_a, lengths_a, K_a, idxs_a, min_point_dist_a, start_idxs_a); + break; + default: + FarthestPointSamplingKernel<1024> + <<>>( + points_a, lengths_a, K_a, idxs_a, min_point_dist_a, start_idxs_a); + } + + AT_CUDA_CHECK(cudaGetLastError()); + return idxs; +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/sample_farthest_points/sample_farthest_points.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/sample_farthest_points/sample_farthest_points.h new file mode 100644 index 0000000000000000000000000000000000000000..7b613d358880936863c2a56b82dee77d93d777f9 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/sample_farthest_points/sample_farthest_points.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once +#include +#include +#include "utils/pytorch3d_cutils.h" + +// Iterative farthest point sampling algorithm [1] to subsample a set of +// K points from a given pointcloud. At each iteration, a point is selected +// which has the largest nearest neighbor distance to any of the +// already selected points. + +// Farthest point sampling provides more uniform coverage of the input +// point cloud compared to uniform random sampling. + +// [1] Charles R. Qi et al, "PointNet++: Deep Hierarchical Feature Learning +// on Point Sets in a Metric Space", NeurIPS 2017. + +// Args: +// points: (N, P, D) float32 Tensor containing the batch of pointclouds. +// lengths: (N,) long Tensor giving the number of points in each pointcloud +// (to support heterogeneous batches of pointclouds). +// K: a tensor of length (N,) giving the number of +// samples to select for each element in the batch. +// The number of samples is typically << P. +// start_idxs: (N,) long Tensor giving the index of the first point to +// sample. Default is all 0. When a random start point is required, +// start_idxs should be set to a random value between [0, lengths[n]] +// for batch element n. +// Returns: +// selected_indices: (N, K) array of selected indices. If the values in +// K are not all the same, then the shape will be (N, max(K), D), and +// padded with -1 for batch elements where k_i < max(K). The selected +// points are gathered in the pytorch autograd wrapper. + +at::Tensor FarthestPointSamplingCuda( + const at::Tensor& points, + const at::Tensor& lengths, + const at::Tensor& K, + const at::Tensor& start_idxs); + +at::Tensor FarthestPointSamplingCpu( + const at::Tensor& points, + const at::Tensor& lengths, + const at::Tensor& K, + const at::Tensor& start_idxs); + +// Exposed implementation. +at::Tensor FarthestPointSampling( + const at::Tensor& points, + const at::Tensor& lengths, + const at::Tensor& K, + const at::Tensor& start_idxs) { + if (points.is_cuda() || lengths.is_cuda() || K.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(points); + CHECK_CUDA(lengths); + CHECK_CUDA(K); + CHECK_CUDA(start_idxs); + return FarthestPointSamplingCuda(points, lengths, K, start_idxs); +#else + AT_ERROR("Not compiled with GPU support."); +#endif + } + return FarthestPointSamplingCpu(points, lengths, K, start_idxs); +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/sample_farthest_points/sample_farthest_points_cpu.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/sample_farthest_points/sample_farthest_points_cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..cd533825f4da75e232bc493c28a0872e477d6db7 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/sample_farthest_points/sample_farthest_points_cpu.cpp @@ -0,0 +1,103 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include +#include + +at::Tensor FarthestPointSamplingCpu( + const at::Tensor& points, + const at::Tensor& lengths, + const at::Tensor& K, + const at::Tensor& start_idxs) { + // Get constants + const int64_t N = points.size(0); + const int64_t P = points.size(1); + const int64_t D = points.size(2); + const int64_t max_K = torch::max(K).item(); + + // Initialize an output array for the sampled indices + // of shape (N, max_K) + auto long_opts = lengths.options(); + torch::Tensor sampled_indices = torch::full({N, max_K}, -1, long_opts); + + // Create accessors for all tensors + auto points_a = points.accessor(); + auto lengths_a = lengths.accessor(); + auto k_a = K.accessor(); + auto sampled_indices_a = sampled_indices.accessor(); + auto start_idxs_a = start_idxs.accessor(); + + // Initialize a mask to prevent duplicates + // If true, the point has already been selected. + std::vector selected_points_mask(P, false); + + // Initialize to infinity a vector of + // distances from each point to any of the previously selected points + std::vector dists(P, std::numeric_limits::max()); + + for (int64_t n = 0; n < N; ++n) { + // Resize and reset points mask and distances for each batch + selected_points_mask.resize(lengths_a[n]); + dists.resize(lengths_a[n]); + std::fill(selected_points_mask.begin(), selected_points_mask.end(), false); + std::fill(dists.begin(), dists.end(), std::numeric_limits::max()); + + // Get the starting point index and save it + int64_t last_idx = start_idxs_a[n]; + sampled_indices_a[n][0] = last_idx; + + // Set the value of the mask at this point to false + selected_points_mask[last_idx] = true; + + // For heterogeneous pointclouds, use the minimum of the + // length for that cloud compared to K as the number of + // points to sample + const int64_t batch_k = std::min(lengths_a[n], k_a[n]); + + // Iteratively select batch_k points per batch + for (int64_t k = 1; k < batch_k; ++k) { + // Iterate through all the points + for (int64_t p = 0; p < lengths_a[n]; ++p) { + if (selected_points_mask[p]) { + // For already selected points set the distance to 0.0 + dists[p] = 0.0; + continue; + } + + // Calculate the distance to the last selected point + float dist2 = 0.0; + for (int64_t d = 0; d < D; ++d) { + float diff = points_a[n][last_idx][d] - points_a[n][p][d]; + dist2 += diff * diff; + } + + // If the distance of this point to the last selected point is closer + // than the distance to any of the previously selected points, then + // update this distance + if (dist2 < dists[p]) { + dists[p] = dist2; + } + } + + // The aim is to pick the point that has the largest + // nearest neighbour distance to any of the already selected points + auto itr = std::max_element(dists.begin(), dists.end()); + last_idx = std::distance(dists.begin(), itr); + + // Save selected point + sampled_indices_a[n][k] = last_idx; + + // Set the mask value to true to prevent duplicates. + selected_points_mask[last_idx] = true; + } + } + + return sampled_indices; +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/sample_pdf/sample_pdf.cu b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/sample_pdf/sample_pdf.cu new file mode 100644 index 0000000000000000000000000000000000000000..885313ac3c8aaba173c4691a1189ea43ddd39f1e --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/sample_pdf/sample_pdf.cu @@ -0,0 +1,153 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include + +// There is no intermediate memory, so no reason not to have blocksize=32. +// 256 is a reasonable number of blocks. + +// DESIGN +// We exploit the fact that n_samples is not tiny. +// A chunk of work is T*blocksize many samples from +// a single batch elememt. +// For each batch element there will be +// chunks_per_batch = 1 + (n_samples-1)/(T*blocksize) of them. +// The number of potential chunks to do is +// n_chunks = chunks_per_batch * n_batches. +// These chunks are divided among the gridSize-many blocks. +// In block b, we work on chunks b, b+gridSize, b+2*gridSize etc . +// In chunk i, we work on batch_element i/chunks_per_batch +// on samples starting from (i%chunks_per_batch) * (T*blocksize) + +// BEGIN HYPOTHETICAL +// Another option (not implemented) if batch_size was always large +// would be as follows. + +// A chunk of work is S samples from each of blocksize-many +// batch elements. +// For each batch element there will be +// chunks_per_batch = (1+(n_samples-1)/S) of them. +// The number of potential chunks to do is +// n_chunks = chunks_per_batch * (1+(n_batches-1)/blocksize) +// These chunks are divided among the gridSize-many blocks. +// In block b, we work on chunks b, b+gridSize, b+2*gridSize etc . +// In chunk i, we work on samples starting from S*(i%chunks_per_batch) +// on batch elements starting from blocksize*(i/chunks_per_batch). +// END HYPOTHETICAL + +__global__ void SamplePdfCudaKernel( + const float* __restrict__ bins, + const float* __restrict__ weights, + float* __restrict__ outputs, + float eps, + const int T, + const int64_t batch_size, + const int64_t n_bins, + const int64_t n_samples) { + const int64_t chunks_per_batch = 1 + (n_samples - 1) / (T * blockDim.x); + const int64_t n_chunks = chunks_per_batch * batch_size; + + for (int64_t i_chunk = blockIdx.x; i_chunk < n_chunks; i_chunk += gridDim.x) { + // Loop over the chunks. + int64_t i_batch_element = i_chunk / chunks_per_batch; + int64_t sample_start = (i_chunk % chunks_per_batch) * (T * blockDim.x); + const float* const weight_startp = weights + n_bins * i_batch_element; + const float* const bin_startp = bins + (1 + n_bins) * i_batch_element; + + // Each chunk looks at a single batch element, so we do the preprocessing + // which depends on the batch element, namely finding the total weight. + // Idenntical work is being done in sync here by every thread of the block. + float total_weight = eps; + for (int64_t i_bin = 0; i_bin < n_bins; ++i_bin) { + total_weight += weight_startp[i_bin]; + } + + float* const output_startp = + outputs + n_samples * i_batch_element + sample_start; + + for (int t = 0; t < T; ++t) { + // Loop over T, which is the number of samples each thread makes within + // the chunk. + const int64_t i_sample_within_chunk = threadIdx.x + t * blockDim.x; + if (sample_start + i_sample_within_chunk >= n_samples) { + // Some threads need to exit early because the sample they would + // make is unwanted. + continue; + } + // output_startp[i_sample_within_chunk] contains the quantile we (i.e. + // this thread) are calcvulating. + float uniform = total_weight * output_startp[i_sample_within_chunk]; + int64_t i_bin = 0; + // We find the bin containing the quantile by walking along the weights. + // This loop must be thread dependent. I.e. the whole warp will wait until + // every thread has found the bin for its quantile. + // It may be best to write it differently. + while (i_bin + 1 < n_bins && uniform > weight_startp[i_bin]) { + uniform -= weight_startp[i_bin]; + ++i_bin; + } + + // Now we know which bin to look in, we use linear interpolation + // to find the location of the quantile within the bin, and + // write the answer back. + float bin_start = bin_startp[i_bin]; + float bin_end = bin_startp[i_bin + 1]; + float bin_weight = weight_startp[i_bin]; + float output_value = bin_start; + if (uniform > bin_weight) { + output_value = bin_end; + } else if (bin_weight > eps) { + output_value += (uniform / bin_weight) * (bin_end - bin_start); + } + output_startp[i_sample_within_chunk] = output_value; + } + } +} + +void SamplePdfCuda( + const at::Tensor& bins, + const at::Tensor& weights, + const at::Tensor& outputs, + float eps) { + // Check inputs are on the same device + at::TensorArg bins_t{bins, "bins", 1}, weights_t{weights, "weights", 2}, + outputs_t{outputs, "outputs", 3}; + at::CheckedFrom c = "SamplePdfCuda"; + at::checkAllSameGPU(c, {bins_t, weights_t, outputs_t}); + at::checkAllSameType(c, {bins_t, weights_t, outputs_t}); + + // Set the device for the kernel launch based on the device of the input + at::cuda::CUDAGuard device_guard(bins.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + const int64_t batch_size = bins.size(0); + const int64_t n_bins = weights.size(1); + const int64_t n_samples = outputs.size(1); + + const int64_t threads = 32; + const int64_t T = n_samples <= threads ? 1 : 2; + const int64_t chunks_per_batch = 1 + (n_samples - 1) / (T * threads); + const int64_t n_chunks = chunks_per_batch * batch_size; + + const int64_t max_blocks = 1024; + const int64_t blocks = n_chunks < max_blocks ? n_chunks : max_blocks; + + SamplePdfCudaKernel<<>>( + bins.contiguous().data_ptr(), + weights.contiguous().data_ptr(), + outputs.data_ptr(), // Checked contiguous in header file. + eps, + T, + batch_size, + n_bins, + n_samples); + + AT_CUDA_CHECK(cudaGetLastError()); +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/sample_pdf/sample_pdf.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/sample_pdf/sample_pdf.h new file mode 100644 index 0000000000000000000000000000000000000000..899117df797cf03c3c207e6205d1607dd3707f3d --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/sample_pdf/sample_pdf.h @@ -0,0 +1,76 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once +#include +#include +#include +#include +#include "utils/pytorch3d_cutils.h" + +// **************************************************************************** +// * SamplePdf * +// **************************************************************************** + +// Samples a probability density functions defined by bin edges `bins` and +// the non-negative per-bin probabilities `weights`. + +// Args: +// bins: FloatTensor of shape `(batch_size, n_bins+1)` denoting the edges +// of the sampling bins. + +// weights: FloatTensor of shape `(batch_size, n_bins)` containing +// non-negative numbers representing the probability of sampling the +// corresponding bin. + +// uniforms: The quantiles to draw, FloatTensor of shape +// `(batch_size, n_samples)`. + +// outputs: On call, this contains the quantiles to draw. It is overwritten +// with the drawn samples. FloatTensor of shape +// `(batch_size, n_samples), where `n_samples are drawn from each +// distribution. + +// eps: A constant preventing division by zero in case empty bins are +// present. + +// Not differentiable + +#ifdef WITH_CUDA +void SamplePdfCuda( + const torch::Tensor& bins, + const torch::Tensor& weights, + const torch::Tensor& outputs, + float eps); +#endif + +void SamplePdfCpu( + const torch::Tensor& bins, + const torch::Tensor& weights, + const torch::Tensor& outputs, + float eps); + +inline void SamplePdf( + const torch::Tensor& bins, + const torch::Tensor& weights, + const torch::Tensor& outputs, + float eps) { + if (bins.is_cuda()) { +#ifdef WITH_CUDA + CHECK_CUDA(weights); + CHECK_CONTIGUOUS_CUDA(outputs); + torch::autograd::increment_version(outputs); + SamplePdfCuda(bins, weights, outputs, eps); + return; +#else + AT_ERROR("Not compiled with GPU support."); +#endif + } + CHECK_CONTIGUOUS(outputs); + SamplePdfCpu(bins, weights, outputs, eps); +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/sample_pdf/sample_pdf_cpu.cpp b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/sample_pdf/sample_pdf_cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..272197c6e9d5f8ee19153004310fb0fd8e10b94b --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/sample_pdf/sample_pdf_cpu.cpp @@ -0,0 +1,142 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include +#include +#include + +// If the number of bins is the typical 64, it is +// quicker to use binary search than linear scan. +// With more bins, it is more important. +// There is no equivalent CUDA implementation yet. +#define USE_BINARY_SEARCH + +namespace { +// This worker function does the job of SamplePdf but only on +// batch elements in [start_batch, end_batch). +void SamplePdfCpu_worker( + const torch::Tensor& bins, + const torch::Tensor& weights, + const torch::Tensor& outputs, + float eps, + int64_t start_batch, + int64_t end_batch) { + const int64_t n_bins = weights.size(1); + const int64_t n_samples = outputs.size(1); + + auto bins_a = bins.accessor(); + auto weights_a = weights.accessor(); + float* output_p = outputs.data_ptr() + start_batch * n_samples; + +#ifdef USE_BINARY_SEARCH + std::vector partial_sums(n_bins); +#endif + + for (int64_t i_batch_elt = start_batch; i_batch_elt < end_batch; + ++i_batch_elt) { + auto bin_a = bins_a[i_batch_elt]; + auto weight_a = weights_a[i_batch_elt]; + + // Here we do the work which has to be done once per batch element. + // i.e. (1) finding the total weight. (2) If using binary search, + // precompute the partial sums of the weights. + + float total_weight = 0; + for (int64_t i_bin = 0; i_bin < n_bins; ++i_bin) { + total_weight += weight_a[i_bin]; +#ifdef USE_BINARY_SEARCH + partial_sums[i_bin] = total_weight; +#endif + } + total_weight += eps; + + for (int64_t i_sample = 0; i_sample < n_samples; ++i_sample) { + // Here we are taking a single random quantile (which is stored + // in *output_p) and using it to make a single sample, which we + // write back to the same location. First we find which bin + // the quantile lives in, either by binary search in the + // precomputed partial sums, or by scanning through the weights. + + float uniform = total_weight * *output_p; +#ifdef USE_BINARY_SEARCH + int64_t i_bin = std::lower_bound( + partial_sums.begin(), --partial_sums.end(), uniform) - + partial_sums.begin(); + if (i_bin > 0) { + uniform -= partial_sums[i_bin - 1]; + } +#else + int64_t i_bin = 0; + while (i_bin + 1 < n_bins && uniform > weight_a[i_bin]) { + uniform -= weight_a[i_bin]; + ++i_bin; + } +#endif + + // Now i_bin identifies the bin the quantile lives in, we use + // straight line interpolation to find the position of the + // quantile within the bin, and write it to *output_p. + + float bin_start = bin_a[i_bin]; + float bin_end = bin_a[i_bin + 1]; + float bin_weight = weight_a[i_bin]; + float output_value = bin_start; + if (uniform > bin_weight) { + output_value = bin_end; + } else if (bin_weight > eps) { + output_value += (uniform / bin_weight) * (bin_end - bin_start); + } + *output_p = output_value; + ++output_p; + } + } +} + +} // anonymous namespace + +void SamplePdfCpu( + const torch::Tensor& bins, + const torch::Tensor& weights, + const torch::Tensor& outputs, + float eps) { + const int64_t batch_size = bins.size(0); + const int64_t max_threads = std::min(4, at::get_num_threads()); + const int64_t n_threads = std::min(max_threads, batch_size); + if (batch_size == 0) { + return; + } + + // SamplePdfCpu_worker does the work of this function. We send separate ranges + // of batch elements to that function in nThreads-1 separate threads. + + std::vector threads; + threads.reserve(n_threads - 1); + const int64_t batch_elements_per_thread = 1 + (batch_size - 1) / n_threads; + int64_t start_batch = 0; + for (int iThread = 0; iThread < n_threads - 1; ++iThread) { + threads.emplace_back( + SamplePdfCpu_worker, + bins, + weights, + outputs, + eps, + start_batch, + start_batch + batch_elements_per_thread); + start_batch += batch_elements_per_thread; + } + + // The remaining batch elements are calculated in this threads. If nThreads is + // 1 then all the work happens in this line. + SamplePdfCpu_worker(bins, weights, outputs, eps, start_batch, batch_size); + for (auto&& thread : threads) { + thread.join(); + } + torch::autograd::increment_version(outputs); +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/dispatch.cuh b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/dispatch.cuh new file mode 100644 index 0000000000000000000000000000000000000000..83f3d69ff40907c396e3d175402d5cf4561142b5 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/dispatch.cuh @@ -0,0 +1,357 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +// This file provides utilities for dispatching to specialized versions of +// functions. This is especially useful for CUDA kernels, since specializing +// them to particular input sizes can often allow the compiler to unroll loops +// and place arrays into registers, which can give huge performance speedups. +// +// As an example, suppose we have the following function which is specialized +// based on a compile-time int64_t value: +// +// template +// struct SquareOffset { +// static void run(T y) { +// T val = x * x + y; +// std::cout << val << std::endl; +// } +// } +// +// This function takes one compile-time argument x, and one run-time argument y. +// We might want to compile specialized versions of this for x=0, x=1, etc and +// then dispatch to the correct one based on the runtime value of x. +// One simple way to achieve this is with a lookup table: +// +// template +// void DispatchSquareOffset(const int64_t x, T y) { +// if (x == 0) { +// SquareOffset::run(y); +// } else if (x == 1) { +// SquareOffset::run(y); +// } else if (x == 2) { +// SquareOffset::run(y); +// } +// } +// +// This function takes both x and y as run-time arguments, and dispatches to +// different specialized versions of SquareOffset based on the run-time value +// of x. This works, but it's tedious and error-prone. If we want to change the +// set of x values for which we provide compile-time specializations, then we +// will need to do a lot of tedius editing of the dispatch function. Also, if we +// want to provide compile-time specializations for another function other than +// SquareOffset, we will need to duplicate the entire lookup table. +// +// To solve these problems, we can use the DispatchKernel1D function provided by +// this file instead: +// +// template +// void DispatchSquareOffset(const int64_t x, T y) { +// constexpr int64_t xmin = 0; +// constexpr int64_t xmax = 2; +// DispatchKernel1D(x, y); +// } +// +// DispatchKernel1D uses template metaprogramming to compile specialized +// versions of SquareOffset for all values of x with xmin <= x <= xmax, and +// then dispatches to the correct one based on the run-time value of x. If we +// want to change the range of x values for which SquareOffset is specialized +// at compile-time, then all we have to do is change the values of the +// compile-time constants xmin and xmax. +// +// This file also allows us to similarly dispatch functions that depend on two +// compile-time int64_t values, using the DispatchKernel2D function like this: +// +// template +// struct Sum { +// static void run(T z, T w) { +// T val = x + y + z + w; +// std::cout << val << std::endl; +// } +// } +// +// template +// void DispatchSum(const int64_t x, const int64_t y, int z, int w) { +// constexpr int64_t xmin = 1; +// constexpr int64_t xmax = 3; +// constexpr int64_t ymin = 2; +// constexpr int64_t ymax = 5; +// DispatchKernel2D(x, y, z, w); +// } +// +// Like its 1D counterpart, DispatchKernel2D uses template metaprogramming to +// compile specialized versions of sum for all values of (x, y) with +// xmin <= x <= xmax and ymin <= y <= ymax, then dispatches to the correct +// specialized version based on the runtime values of x and y. + +// Define some helper structs in an anonymous namespace. +namespace { + +// 1D dispatch: general case. +// Kernel is the function we want to dispatch to; it should take a typename and +// an int64_t as template args, and it should define a static void function +// run which takes any number of arguments of any type. +// In order to dispatch, we will take an additional template argument curN, +// and increment it via template recursion until it is equal to the run-time +// argument N. +template < + template + class Kernel, + typename T, + int64_t minN, + int64_t maxN, + int64_t curN, + typename... Args> +struct DispatchKernelHelper1D { + static void run(const int64_t N, Args... args) { + if (curN == N) { + // The compile-time value curN is equal to the run-time value N, so we + // can dispatch to the run method of the Kernel. + Kernel::run(args...); + } else if (curN < N) { + // Increment curN via template recursion + DispatchKernelHelper1D::run( + N, args...); + } + // We shouldn't get here -- throw an error? + } +}; + +// 1D dispatch: Specialization when curN == maxN +// We need this base case to avoid infinite template recursion. +template < + template + class Kernel, + typename T, + int64_t minN, + int64_t maxN, + typename... Args> +struct DispatchKernelHelper1D { + static void run(const int64_t N, Args... args) { + if (N == maxN) { + Kernel::run(args...); + } + // We shouldn't get here -- throw an error? + } +}; + +// 2D dispatch, general case. +// This is similar to the 1D case: we take additional template args curN and +// curM, and increment them via template recursion until they are equal to +// the run-time values of N and M, at which point we dispatch to the run +// method of the kernel. +template < + template + class Kernel, + typename T, + int64_t minN, + int64_t maxN, + int64_t curN, + int64_t minM, + int64_t maxM, + int64_t curM, + typename... Args> +struct DispatchKernelHelper2D { + static void run(const int64_t N, const int64_t M, Args... args) { + if (curN == N && curM == M) { + Kernel::run(args...); + } else if (curN < N && curM < M) { + // Increment both curN and curM. This isn't strictly necessary; we could + // just increment one or the other at each step. But this helps to cut + // on the number of recursive calls we make. + DispatchKernelHelper2D< + Kernel, + T, + minN, + maxN, + curN + 1, + minM, + maxM, + curM + 1, + Args...>::run(N, M, args...); + } else if (curN < N) { + // Increment curN only + DispatchKernelHelper2D< + Kernel, + T, + minN, + maxN, + curN + 1, + minM, + maxM, + curM, + Args...>::run(N, M, args...); + } else if (curM < M) { + // Increment curM only + DispatchKernelHelper2D< + Kernel, + T, + minN, + maxN, + curN, + minM, + maxM, + curM + 1, + Args...>::run(N, M, args...); + } + } +}; + +// 2D dispatch, specialization for curN == maxN +template < + template + class Kernel, + typename T, + int64_t minN, + int64_t maxN, + int64_t minM, + int64_t maxM, + int64_t curM, + typename... Args> +struct DispatchKernelHelper2D< + Kernel, + T, + minN, + maxN, + maxN, + minM, + maxM, + curM, + Args...> { + static void run(const int64_t N, const int64_t M, Args... args) { + if (maxN == N && curM == M) { + Kernel::run(args...); + } else if (curM < maxM) { + DispatchKernelHelper2D< + Kernel, + T, + minN, + maxN, + maxN, + minM, + maxM, + curM + 1, + Args...>::run(N, M, args...); + } + // We should not get here -- throw an error? + } +}; + +// 2D dispatch, specialization for curM == maxM +template < + template + class Kernel, + typename T, + int64_t minN, + int64_t maxN, + int64_t curN, + int64_t minM, + int64_t maxM, + typename... Args> +struct DispatchKernelHelper2D< + Kernel, + T, + minN, + maxN, + curN, + minM, + maxM, + maxM, + Args...> { + static void run(const int64_t N, const int64_t M, Args... args) { + if (curN == N && maxM == M) { + Kernel::run(args...); + } else if (curN < maxN) { + DispatchKernelHelper2D< + Kernel, + T, + minN, + maxN, + curN + 1, + minM, + maxM, + maxM, + Args...>::run(N, M, args...); + } + // We should not get here -- throw an error? + } +}; + +// 2D dispatch, specialization for curN == maxN, curM == maxM +template < + template + class Kernel, + typename T, + int64_t minN, + int64_t maxN, + int64_t minM, + int64_t maxM, + typename... Args> +struct DispatchKernelHelper2D< + Kernel, + T, + minN, + maxN, + maxN, + minM, + maxM, + maxM, + Args...> { + static void run(const int64_t N, const int64_t M, Args... args) { + if (maxN == N && maxM == M) { + Kernel::run(args...); + } + // We should not get here -- throw an error? + } +}; + +} // namespace + +// This is the function we expect users to call to dispatch to 1D functions +template < + template + class Kernel, + typename T, + int64_t minN, + int64_t maxN, + typename... Args> +void DispatchKernel1D(const int64_t N, Args... args) { + if (minN <= N && N <= maxN) { + // Kick off the template recursion by calling the Helper with curN = minN + DispatchKernelHelper1D::run( + N, args...); + } + // Maybe throw an error if we tried to dispatch outside the allowed range? +} + +// This is the function we expect users to call to dispatch to 2D functions +template < + template + class Kernel, + typename T, + int64_t minN, + int64_t maxN, + int64_t minM, + int64_t maxM, + typename... Args> +void DispatchKernel2D(const int64_t N, const int64_t M, Args... args) { + if (minN <= N && N <= maxN && minM <= M && M <= maxM) { + // Kick off the template recursion by calling the Helper with curN = minN + // and curM = minM + DispatchKernelHelper2D< + Kernel, + T, + minN, + maxN, + minN, + minM, + maxM, + minM, + Args...>::run(N, M, args...); + } + // Maybe throw an error if we tried to dispatch outside the specified range? +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/float_math.cuh b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/float_math.cuh new file mode 100644 index 0000000000000000000000000000000000000000..e48e960e96544fd901655ce1d0217513d300187b --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/float_math.cuh @@ -0,0 +1,153 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once +#include + +// Set epsilon +#ifdef _MSC_VER +#define vEpsilon 1e-8f +#else +const auto vEpsilon = 1e-8; +#endif + +// Common functions and operators for float2. + +__device__ inline float2 operator-(const float2& a, const float2& b) { + return make_float2(a.x - b.x, a.y - b.y); +} + +__device__ inline float2 operator+(const float2& a, const float2& b) { + return make_float2(a.x + b.x, a.y + b.y); +} + +__device__ inline float2 operator/(const float2& a, const float2& b) { + return make_float2(a.x / b.x, a.y / b.y); +} + +__device__ inline float2 operator/(const float2& a, const float b) { + return make_float2(a.x / b, a.y / b); +} + +__device__ inline float2 operator*(const float2& a, const float2& b) { + return make_float2(a.x * b.x, a.y * b.y); +} + +__device__ inline float2 operator*(const float a, const float2& b) { + return make_float2(a * b.x, a * b.y); +} + +__device__ inline float FloatMin3(const float a, const float b, const float c) { + return fminf(a, fminf(b, c)); +} + +__device__ inline float FloatMax3(const float a, const float b, const float c) { + return fmaxf(a, fmaxf(b, c)); +} + +__device__ inline float dot(const float2& a, const float2& b) { + return a.x * b.x + a.y * b.y; +} + +// Backward pass for the dot product. +// Args: +// a, b: Coordinates of two points. +// grad_dot: Upstream gradient for the output. +// +// Returns: +// tuple of gradients for each of the input points: +// (float2 grad_a, float2 grad_b) +// +__device__ inline thrust::tuple +DotBackward(const float2& a, const float2& b, const float& grad_dot) { + return thrust::make_tuple(grad_dot * b, grad_dot * a); +} + +__device__ inline float sum(const float2& a) { + return a.x + a.y; +} + +// Common functions and operators for float3. + +__device__ inline float3 operator-(const float3& a, const float3& b) { + return make_float3(a.x - b.x, a.y - b.y, a.z - b.z); +} + +__device__ inline float3 operator+(const float3& a, const float3& b) { + return make_float3(a.x + b.x, a.y + b.y, a.z + b.z); +} + +__device__ inline float3 operator/(const float3& a, const float3& b) { + return make_float3(a.x / b.x, a.y / b.y, a.z / b.z); +} + +__device__ inline float3 operator/(const float3& a, const float b) { + return make_float3(a.x / b, a.y / b, a.z / b); +} + +__device__ inline float3 operator*(const float3& a, const float3& b) { + return make_float3(a.x * b.x, a.y * b.y, a.z * b.z); +} + +__device__ inline float3 operator*(const float a, const float3& b) { + return make_float3(a * b.x, a * b.y, a * b.z); +} + +__device__ inline float dot(const float3& a, const float3& b) { + return a.x * b.x + a.y * b.y + a.z * b.z; +} + +__device__ inline float sum(const float3& a) { + return a.x + a.y + a.z; +} + +__device__ inline float3 cross(const float3& a, const float3& b) { + return make_float3( + a.y * b.z - a.z * b.y, a.z * b.x - a.x * b.z, a.x * b.y - a.y * b.x); +} + +__device__ inline thrust::tuple +cross_backward(const float3& a, const float3& b, const float3& grad_cross) { + const float grad_ax = -grad_cross.y * b.z + grad_cross.z * b.y; + const float grad_ay = grad_cross.x * b.z - grad_cross.z * b.x; + const float grad_az = -grad_cross.x * b.y + grad_cross.y * b.x; + const float3 grad_a = make_float3(grad_ax, grad_ay, grad_az); + + const float grad_bx = grad_cross.y * a.z - grad_cross.z * a.y; + const float grad_by = -grad_cross.x * a.z + grad_cross.z * a.x; + const float grad_bz = grad_cross.x * a.y - grad_cross.y * a.x; + const float3 grad_b = make_float3(grad_bx, grad_by, grad_bz); + + return thrust::make_tuple(grad_a, grad_b); +} + +__device__ inline float norm(const float3& a) { + return sqrt(dot(a, a)); +} + +__device__ inline float3 normalize(const float3& a) { + return a / (norm(a) + vEpsilon); +} + +__device__ inline float3 normalize_backward( + const float3& a, + const float3& grad_normz) { + const float a_norm = norm(a) + vEpsilon; + const float3 out = a / a_norm; + + const float grad_ax = grad_normz.x * (1.0f - out.x * out.x) / a_norm + + grad_normz.y * (-out.x * out.y) / a_norm + + grad_normz.z * (-out.x * out.z) / a_norm; + const float grad_ay = grad_normz.x * (-out.x * out.y) / a_norm + + grad_normz.y * (1.0f - out.y * out.y) / a_norm + + grad_normz.z * (-out.y * out.z) / a_norm; + const float grad_az = grad_normz.x * (-out.x * out.z) / a_norm + + grad_normz.y * (-out.y * out.z) / a_norm + + grad_normz.z * (1.0f - out.z * out.z) / a_norm; + return make_float3(grad_ax, grad_ay, grad_az); +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/geometry_utils.cuh b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/geometry_utils.cuh new file mode 100644 index 0000000000000000000000000000000000000000..66aee7fc7bcd3495bc7dbba56d89995d383b655e --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/geometry_utils.cuh @@ -0,0 +1,792 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include +#include "float_math.cuh" + +// Set epsilon for preventing floating point errors and division by 0. +#ifdef _MSC_VER +#define kEpsilon 1e-8f +#else +const auto kEpsilon = 1e-8; +#endif + +// ************************************************************* // +// vec2 utils // +// ************************************************************* // + +// Determines whether a point p is on the right side of a 2D line segment +// given by the end points v0, v1. +// +// Args: +// p: vec2 Coordinates of a point. +// v0, v1: vec2 Coordinates of the end points of the edge. +// +// Returns: +// area: The signed area of the parallelogram given by the vectors +// A = p - v0 +// B = v1 - v0 +// +__device__ inline float +EdgeFunctionForward(const float2& p, const float2& v0, const float2& v1) { + return (p.x - v0.x) * (v1.y - v0.y) - (p.y - v0.y) * (v1.x - v0.x); +} + +// Backward pass for the edge function returning partial dervivatives for each +// of the input points. +// +// Args: +// p: vec2 Coordinates of a point. +// v0, v1: vec2 Coordinates of the end points of the edge. +// grad_edge: Upstream gradient for output from edge function. +// +// Returns: +// tuple of gradients for each of the input points: +// (float2 d_edge_dp, float2 d_edge_dv0, float2 d_edge_dv1) +// +__device__ inline thrust::tuple EdgeFunctionBackward( + const float2& p, + const float2& v0, + const float2& v1, + const float& grad_edge) { + const float2 dedge_dp = make_float2(v1.y - v0.y, v0.x - v1.x); + const float2 dedge_dv0 = make_float2(p.y - v1.y, v1.x - p.x); + const float2 dedge_dv1 = make_float2(v0.y - p.y, p.x - v0.x); + return thrust::make_tuple( + grad_edge * dedge_dp, grad_edge * dedge_dv0, grad_edge * dedge_dv1); +} + +// The forward pass for computing the barycentric coordinates of a point +// relative to a triangle. +// +// Args: +// p: Coordinates of a point. +// v0, v1, v2: Coordinates of the triangle vertices. +// +// Returns +// bary: (w0, w1, w2) barycentric coordinates in the range [0, 1]. +// +__device__ inline float3 BarycentricCoordsForward( + const float2& p, + const float2& v0, + const float2& v1, + const float2& v2) { + const float area = EdgeFunctionForward(v2, v0, v1) + kEpsilon; + const float w0 = EdgeFunctionForward(p, v1, v2) / area; + const float w1 = EdgeFunctionForward(p, v2, v0) / area; + const float w2 = EdgeFunctionForward(p, v0, v1) / area; + return make_float3(w0, w1, w2); +} + +// The backward pass for computing the barycentric coordinates of a point +// relative to a triangle. +// +// Args: +// p: Coordinates of a point. +// v0, v1, v2: (x, y) coordinates of the triangle vertices. +// grad_bary_upstream: vec3 Upstream gradient for each of the +// barycentric coordaintes [grad_w0, grad_w1, grad_w2]. +// +// Returns +// tuple of gradients for each of the triangle vertices: +// (float2 grad_v0, float2 grad_v1, float2 grad_v2) +// +__device__ inline thrust::tuple +BarycentricCoordsBackward( + const float2& p, + const float2& v0, + const float2& v1, + const float2& v2, + const float3& grad_bary_upstream) { + const float area = EdgeFunctionForward(v2, v0, v1) + kEpsilon; + const float area2 = pow(area, 2.0f); + const float e0 = EdgeFunctionForward(p, v1, v2); + const float e1 = EdgeFunctionForward(p, v2, v0); + const float e2 = EdgeFunctionForward(p, v0, v1); + + const float grad_w0 = grad_bary_upstream.x; + const float grad_w1 = grad_bary_upstream.y; + const float grad_w2 = grad_bary_upstream.z; + + // Calculate component of the gradient from each of w0, w1 and w2. + // e.g. for w0: + // dloss/dw0_v = dl/dw0 * dw0/dw0_top * dw0_top/dv + // + dl/dw0 * dw0/dw0_bot * dw0_bot/dv + const float dw0_darea = -e0 / (area2); + const float dw0_e0 = 1 / area; + const float dloss_d_w0area = grad_w0 * dw0_darea; + const float dloss_e0 = grad_w0 * dw0_e0; + auto de0_dv = EdgeFunctionBackward(p, v1, v2, dloss_e0); + auto dw0area_dv = EdgeFunctionBackward(v2, v0, v1, dloss_d_w0area); + const float2 dw0_p = thrust::get<0>(de0_dv); + const float2 dw0_dv0 = thrust::get<1>(dw0area_dv); + const float2 dw0_dv1 = thrust::get<1>(de0_dv) + thrust::get<2>(dw0area_dv); + const float2 dw0_dv2 = thrust::get<2>(de0_dv) + thrust::get<0>(dw0area_dv); + + const float dw1_darea = -e1 / (area2); + const float dw1_e1 = 1 / area; + const float dloss_d_w1area = grad_w1 * dw1_darea; + const float dloss_e1 = grad_w1 * dw1_e1; + auto de1_dv = EdgeFunctionBackward(p, v2, v0, dloss_e1); + auto dw1area_dv = EdgeFunctionBackward(v2, v0, v1, dloss_d_w1area); + const float2 dw1_p = thrust::get<0>(de1_dv); + const float2 dw1_dv0 = thrust::get<2>(de1_dv) + thrust::get<1>(dw1area_dv); + const float2 dw1_dv1 = thrust::get<2>(dw1area_dv); + const float2 dw1_dv2 = thrust::get<1>(de1_dv) + thrust::get<0>(dw1area_dv); + + const float dw2_darea = -e2 / (area2); + const float dw2_e2 = 1 / area; + const float dloss_d_w2area = grad_w2 * dw2_darea; + const float dloss_e2 = grad_w2 * dw2_e2; + auto de2_dv = EdgeFunctionBackward(p, v0, v1, dloss_e2); + auto dw2area_dv = EdgeFunctionBackward(v2, v0, v1, dloss_d_w2area); + const float2 dw2_p = thrust::get<0>(de2_dv); + const float2 dw2_dv0 = thrust::get<1>(de2_dv) + thrust::get<1>(dw2area_dv); + const float2 dw2_dv1 = thrust::get<2>(de2_dv) + thrust::get<2>(dw2area_dv); + const float2 dw2_dv2 = thrust::get<0>(dw2area_dv); + + const float2 dbary_p = dw0_p + dw1_p + dw2_p; + const float2 dbary_dv0 = dw0_dv0 + dw1_dv0 + dw2_dv0; + const float2 dbary_dv1 = dw0_dv1 + dw1_dv1 + dw2_dv1; + const float2 dbary_dv2 = dw0_dv2 + dw1_dv2 + dw2_dv2; + + return thrust::make_tuple(dbary_p, dbary_dv0, dbary_dv1, dbary_dv2); +} + +// Forward pass for applying perspective correction to barycentric coordinates. +// +// Args: +// bary: Screen-space barycentric coordinates for a point +// z0, z1, z2: Camera-space z-coordinates of the triangle vertices +// +// Returns +// World-space barycentric coordinates +// +__device__ inline float3 BarycentricPerspectiveCorrectionForward( + const float3& bary, + const float z0, + const float z1, + const float z2) { + const float w0_top = bary.x * z1 * z2; + const float w1_top = z0 * bary.y * z2; + const float w2_top = z0 * z1 * bary.z; + const float denom = fmaxf(w0_top + w1_top + w2_top, kEpsilon); + const float w0 = w0_top / denom; + const float w1 = w1_top / denom; + const float w2 = w2_top / denom; + return make_float3(w0, w1, w2); +} + +// Backward pass for applying perspective correction to barycentric coordinates. +// +// Args: +// bary: Screen-space barycentric coordinates for a point +// z0, z1, z2: Camera-space z-coordinates of the triangle vertices +// grad_out: Upstream gradient of the loss with respect to the corrected +// barycentric coordinates. +// +// Returns a tuple of: +// grad_bary: Downstream gradient of the loss with respect to the the +// uncorrected barycentric coordinates. +// grad_z0, grad_z1, grad_z2: Downstream gradient of the loss with respect +// to the z-coordinates of the triangle verts +__device__ inline thrust::tuple +BarycentricPerspectiveCorrectionBackward( + const float3& bary, + const float z0, + const float z1, + const float z2, + const float3& grad_out) { + // Recompute forward pass + const float w0_top = bary.x * z1 * z2; + const float w1_top = z0 * bary.y * z2; + const float w2_top = z0 * z1 * bary.z; + const float denom = fmaxf(w0_top + w1_top + w2_top, kEpsilon); + + // Now do backward pass + const float grad_denom_top = + -w0_top * grad_out.x - w1_top * grad_out.y - w2_top * grad_out.z; + const float grad_denom = grad_denom_top / (denom * denom); + const float grad_w0_top = grad_denom + grad_out.x / denom; + const float grad_w1_top = grad_denom + grad_out.y / denom; + const float grad_w2_top = grad_denom + grad_out.z / denom; + const float grad_bary_x = grad_w0_top * z1 * z2; + const float grad_bary_y = grad_w1_top * z0 * z2; + const float grad_bary_z = grad_w2_top * z0 * z1; + const float3 grad_bary = make_float3(grad_bary_x, grad_bary_y, grad_bary_z); + const float grad_z0 = grad_w1_top * bary.y * z2 + grad_w2_top * bary.z * z1; + const float grad_z1 = grad_w0_top * bary.x * z2 + grad_w2_top * bary.z * z0; + const float grad_z2 = grad_w0_top * bary.x * z1 + grad_w1_top * bary.y * z0; + return thrust::make_tuple(grad_bary, grad_z0, grad_z1, grad_z2); +} + +// Clip negative barycentric coordinates to 0.0 and renormalize so +// the barycentric coordinates for a point sum to 1. When the blur_radius +// is greater than 0, a face will still be recorded as overlapping a pixel +// if the pixel is outside the face. In this case at least one of the +// barycentric coordinates for the pixel relative to the face will be negative. +// Clipping will ensure that the texture and z buffer are interpolated +// correctly. +// +// Args +// bary: (w0, w1, w2) barycentric coordinates which can be outside the +// range [0, 1]. +// +// Returns +// bary: (w0, w1, w2) barycentric coordinates in the range [0, 1] which +// satisfy the condition: sum(w0, w1, w2) = 1.0. +// +__device__ inline float3 BarycentricClipForward(const float3 bary) { + float3 w = make_float3(0.0f, 0.0f, 0.0f); + // Clamp lower bound only + w.x = max(bary.x, 0.0); + w.y = max(bary.y, 0.0); + w.z = max(bary.z, 0.0); + float w_sum = w.x + w.y + w.z; + w_sum = fmaxf(w_sum, 1e-5); + w.x /= w_sum; + w.y /= w_sum; + w.z /= w_sum; + + return w; +} + +// Backward pass for barycentric coordinate clipping. +// +// Args +// bary: (w0, w1, w2) barycentric coordinates which can be outside the +// range [0, 1]. +// grad_baryclip_upstream: vec3 Upstream gradient for each of the clipped +// barycentric coordinates [grad_w0, grad_w1, grad_w2]. +// +// Returns +// vec3 of gradients for the unclipped barycentric coordinates: +// (grad_w0, grad_w1, grad_w2) +// +__device__ inline float3 BarycentricClipBackward( + const float3 bary, + const float3 grad_baryclip_upstream) { + // Redo some of the forward pass calculations + float3 w = make_float3(0.0f, 0.0f, 0.0f); + // Clamp lower bound only + w.x = max(bary.x, 0.0); + w.y = max(bary.y, 0.0); + w.z = max(bary.z, 0.0); + float w_sum = w.x + w.y + w.z; + + float3 grad_bary = make_float3(1.0f, 1.0f, 1.0f); + float3 grad_clip = make_float3(1.0f, 1.0f, 1.0f); + float3 grad_sum = make_float3(1.0f, 1.0f, 1.0f); + + // Check if sum was clipped. + float grad_sum_clip = 1.0f; + if (w_sum < 1e-5) { + grad_sum_clip = 0.0f; + w_sum = 1e-5; + } + + // Check if any of bary values have been clipped. + if (bary.x < 0.0f) { + grad_clip.x = 0.0f; + } + if (bary.y < 0.0f) { + grad_clip.y = 0.0f; + } + if (bary.z < 0.0f) { + grad_clip.z = 0.0f; + } + + // Gradients of the sum. + grad_sum.x = -w.x / (pow(w_sum, 2.0f)) * grad_sum_clip; + grad_sum.y = -w.y / (pow(w_sum, 2.0f)) * grad_sum_clip; + grad_sum.z = -w.z / (pow(w_sum, 2.0f)) * grad_sum_clip; + + // Gradients for each of the bary coordinates including the cross terms + // from the sum. + grad_bary.x = grad_clip.x * + (grad_baryclip_upstream.x * (1.0f / w_sum + grad_sum.x) + + grad_baryclip_upstream.y * (grad_sum.y) + + grad_baryclip_upstream.z * (grad_sum.z)); + + grad_bary.y = grad_clip.y * + (grad_baryclip_upstream.y * (1.0f / w_sum + grad_sum.y) + + grad_baryclip_upstream.x * (grad_sum.x) + + grad_baryclip_upstream.z * (grad_sum.z)); + + grad_bary.z = grad_clip.z * + (grad_baryclip_upstream.z * (1.0f / w_sum + grad_sum.z) + + grad_baryclip_upstream.x * (grad_sum.x) + + grad_baryclip_upstream.y * (grad_sum.y)); + + return grad_bary; +} + +// Return minimum distance between line segment (v1 - v0) and point p. +// +// Args: +// p: Coordinates of a point. +// v0, v1: Coordinates of the end points of the line segment. +// +// Returns: +// squared distance to the boundary of the triangle. +// +__device__ inline float +PointLineDistanceForward(const float2& p, const float2& a, const float2& b) { + const float2 ba = b - a; + float l2 = dot(ba, ba); + float t = dot(ba, p - a) / l2; + if (l2 <= kEpsilon) { + return dot(p - b, p - b); + } + t = __saturatef(t); // clamp to the interval [+0.0, 1.0] + const float2 p_proj = a + t * ba; + const float2 d = (p_proj - p); + return dot(d, d); // squared distance +} + +// Backward pass for point to line distance in 2D. +// +// Args: +// p: Coordinates of a point. +// v0, v1: Coordinates of the end points of the line segment. +// grad_dist: Upstream gradient for the distance. +// +// Returns: +// tuple of gradients for each of the input points: +// (float2 grad_p, float2 grad_v0, float2 grad_v1) +// +__device__ inline thrust::tuple +PointLineDistanceBackward( + const float2& p, + const float2& v0, + const float2& v1, + const float& grad_dist) { + // Redo some of the forward pass calculations. + const float2 v1v0 = v1 - v0; + const float2 pv0 = p - v0; + const float t_bot = dot(v1v0, v1v0); + const float t_top = dot(v1v0, pv0); + float tt = t_top / t_bot; + tt = __saturatef(tt); + const float2 p_proj = (1.0f - tt) * v0 + tt * v1; + const float2 d = p - p_proj; + const float dist = sqrt(dot(d, d)); + + const float2 grad_p = -1.0f * grad_dist * 2.0f * (p_proj - p); + const float2 grad_v0 = grad_dist * (1.0f - tt) * 2.0f * (p_proj - p); + const float2 grad_v1 = grad_dist * tt * 2.0f * (p_proj - p); + + return thrust::make_tuple(grad_p, grad_v0, grad_v1); +} + +// The forward pass for calculating the shortest distance between a point +// and a triangle. +// +// Args: +// p: Coordinates of a point. +// v0, v1, v2: Coordinates of the three triangle vertices. +// +// Returns: +// shortest squared distance from a point to a triangle. +// +__device__ inline float PointTriangleDistanceForward( + const float2& p, + const float2& v0, + const float2& v1, + const float2& v2) { + // Compute distance to all 3 edges of the triangle and return the min. + const float e01_dist = PointLineDistanceForward(p, v0, v1); + const float e02_dist = PointLineDistanceForward(p, v0, v2); + const float e12_dist = PointLineDistanceForward(p, v1, v2); + const float edge_dist = fminf(fminf(e01_dist, e02_dist), e12_dist); + return edge_dist; +} + +// Backward pass for point triangle distance. +// +// Args: +// p: Coordinates of a point. +// v0, v1, v2: Coordinates of the three triangle vertices. +// grad_dist: Upstream gradient for the distance. +// +// Returns: +// tuple of gradients for each of the triangle vertices: +// (float2 grad_v0, float2 grad_v1, float2 grad_v2) +// +__device__ inline thrust::tuple +PointTriangleDistanceBackward( + const float2& p, + const float2& v0, + const float2& v1, + const float2& v2, + const float& grad_dist) { + // Compute distance to all 3 edges of the triangle. + const float e01_dist = PointLineDistanceForward(p, v0, v1); + const float e02_dist = PointLineDistanceForward(p, v0, v2); + const float e12_dist = PointLineDistanceForward(p, v1, v2); + + // Initialize output tensors. + float2 grad_v0 = make_float2(0.0f, 0.0f); + float2 grad_v1 = make_float2(0.0f, 0.0f); + float2 grad_v2 = make_float2(0.0f, 0.0f); + float2 grad_p = make_float2(0.0f, 0.0f); + + // Find which edge is the closest and return PointLineDistanceBackward for + // that edge. + if (e01_dist <= e02_dist && e01_dist <= e12_dist) { + // Closest edge is v1 - v0. + auto grad_e01 = PointLineDistanceBackward(p, v0, v1, grad_dist); + grad_p = thrust::get<0>(grad_e01); + grad_v0 = thrust::get<1>(grad_e01); + grad_v1 = thrust::get<2>(grad_e01); + } else if (e02_dist <= e01_dist && e02_dist <= e12_dist) { + // Closest edge is v2 - v0. + auto grad_e02 = PointLineDistanceBackward(p, v0, v2, grad_dist); + grad_p = thrust::get<0>(grad_e02); + grad_v0 = thrust::get<1>(grad_e02); + grad_v2 = thrust::get<2>(grad_e02); + } else if (e12_dist <= e01_dist && e12_dist <= e02_dist) { + // Closest edge is v2 - v1. + auto grad_e12 = PointLineDistanceBackward(p, v1, v2, grad_dist); + grad_p = thrust::get<0>(grad_e12); + grad_v1 = thrust::get<1>(grad_e12); + grad_v2 = thrust::get<2>(grad_e12); + } + + return thrust::make_tuple(grad_p, grad_v0, grad_v1, grad_v2); +} + +// ************************************************************* // +// vec3 utils // +// ************************************************************* // + +// Computes the area of a triangle (v0, v1, v2). +// +// Args: +// v0, v1, v2: vec3 coordinates of the triangle vertices +// +// Returns +// area: float: The area of the triangle +// +__device__ inline float +AreaOfTriangle(const float3& v0, const float3& v1, const float3& v2) { + float3 p0 = v1 - v0; + float3 p1 = v2 - v0; + + // compute the hypotenus of the scross product (p0 x p1) + float dd = hypot( + p0.y * p1.z - p0.z * p1.y, + hypot(p0.z * p1.x - p0.x * p1.z, p0.x * p1.y - p0.y * p1.x)); + + return dd / 2.0; +} + +// Computes the barycentric coordinates of a point p relative +// to a triangle (v0, v1, v2), i.e. p = w0 * v0 + w1 * v1 + w2 * v2 +// s.t. w0 + w1 + w2 = 1.0 +// +// NOTE that this function assumes that p lives on the space spanned +// by (v0, v1, v2). +// TODO(gkioxari) explicitly check whether p is coplanar with (v0, v1, v2) +// and throw an error if check fails +// +// Args: +// p: vec3 coordinates of a point +// v0, v1, v2: vec3 coordinates of the triangle vertices +// +// Returns +// bary: (w0, w1, w2) barycentric coordinates +// +__device__ inline float3 BarycentricCoords3Forward( + const float3& p, + const float3& v0, + const float3& v1, + const float3& v2) { + float3 p0 = v1 - v0; + float3 p1 = v2 - v0; + float3 p2 = p - v0; + + const float d00 = dot(p0, p0); + const float d01 = dot(p0, p1); + const float d11 = dot(p1, p1); + const float d20 = dot(p2, p0); + const float d21 = dot(p2, p1); + + const float denom = d00 * d11 - d01 * d01 + kEpsilon; + const float w1 = (d11 * d20 - d01 * d21) / denom; + const float w2 = (d00 * d21 - d01 * d20) / denom; + const float w0 = 1.0f - w1 - w2; + + return make_float3(w0, w1, w2); +} + +// Checks whether the point p is inside the triangle (v0, v1, v2). +// A point is inside the triangle, if all barycentric coordinates +// wrt the triangle are >= 0 & <= 1. +// If the triangle is degenerate, aka line or point, then return False. +// +// NOTE that this function assumes that p lives on the space spanned +// by (v0, v1, v2). +// TODO(gkioxari) explicitly check whether p is coplanar with (v0, v1, v2) +// and throw an error if check fails +// +// Args: +// p: vec3 coordinates of a point +// v0, v1, v2: vec3 coordinates of the triangle vertices +// min_triangle_area: triangles less than this size are considered +// points/lines, IsInsideTriangle returns False +// +// Returns: +// inside: bool indicating wether p is inside triangle +// +__device__ inline bool IsInsideTriangle( + const float3& p, + const float3& v0, + const float3& v1, + const float3& v2, + const double min_triangle_area) { + bool inside; + if (AreaOfTriangle(v0, v1, v2) < min_triangle_area) { + inside = 0; + } else { + float3 bary = BarycentricCoords3Forward(p, v0, v1, v2); + bool x_in = 0.0f <= bary.x && bary.x <= 1.0f; + bool y_in = 0.0f <= bary.y && bary.y <= 1.0f; + bool z_in = 0.0f <= bary.z && bary.z <= 1.0f; + inside = x_in && y_in && z_in; + } + return inside; +} + +// Computes the minimum squared Euclidean distance between the point p +// and the segment spanned by (v0, v1). +// To find this we parametrize p as: x(t) = v0 + t * (v1 - v0) +// and find t which minimizes (x(t) - p) ^ 2. +// Note that p does not need to live in the space spanned by (v0, v1) +// +// Args: +// p: vec3 coordinates of a point +// v0, v1: vec3 coordinates of start and end of segment +// +// Returns: +// dist: the minimum squared distance of p from segment (v0, v1) +// + +__device__ inline float +PointLine3DistanceForward(const float3& p, const float3& v0, const float3& v1) { + const float3 v1v0 = v1 - v0; + const float3 pv0 = p - v0; + const float t_bot = dot(v1v0, v1v0); + const float t_top = dot(pv0, v1v0); + // if t_bot small, then v0 == v1, set tt to 0. + float tt = (t_bot < kEpsilon) ? 0.0f : (t_top / t_bot); + + tt = __saturatef(tt); // clamps to [0, 1] + + const float3 p_proj = v0 + tt * v1v0; + const float3 diff = p - p_proj; + const float dist = dot(diff, diff); + return dist; +} + +// Backward function of the minimum squared Euclidean distance between the point +// p and the line segment (v0, v1). +// +// Args: +// p: vec3 coordinates of a point +// v0, v1: vec3 coordinates of start and end of segment +// grad_dist: Float of the gradient wrt dist +// +// Returns: +// tuple of gradients for the point and line segment (v0, v1): +// (float3 grad_p, float3 grad_v0, float3 grad_v1) + +__device__ inline thrust::tuple +PointLine3DistanceBackward( + const float3& p, + const float3& v0, + const float3& v1, + const float& grad_dist) { + const float3 v1v0 = v1 - v0; + const float3 pv0 = p - v0; + const float t_bot = dot(v1v0, v1v0); + const float t_top = dot(v1v0, pv0); + + float3 grad_p = make_float3(0.0f, 0.0f, 0.0f); + float3 grad_v0 = make_float3(0.0f, 0.0f, 0.0f); + float3 grad_v1 = make_float3(0.0f, 0.0f, 0.0f); + + const float tt = t_top / t_bot; + + if (t_bot < kEpsilon) { + // if t_bot small, then v0 == v1, + // and dist = 0.5 * dot(pv0, pv0) + 0.5 * dot(pv1, pv1) + grad_p = grad_dist * 2.0f * pv0; + grad_v0 = -0.5f * grad_p; + grad_v1 = grad_v0; + } else if (tt < 0.0f) { + grad_p = grad_dist * 2.0f * pv0; + grad_v0 = -1.0f * grad_p; + // no gradients wrt v1 + } else if (tt > 1.0f) { + grad_p = grad_dist * 2.0f * (p - v1); + grad_v1 = -1.0f * grad_p; + // no gradients wrt v0 + } else { + const float3 p_proj = v0 + tt * v1v0; + const float3 diff = p - p_proj; + const float3 grad_base = grad_dist * 2.0f * diff; + grad_p = grad_base - dot(grad_base, v1v0) * v1v0 / t_bot; + const float3 dtt_v0 = (-1.0f * v1v0 - pv0 + 2.0f * tt * v1v0) / t_bot; + grad_v0 = (-1.0f + tt) * grad_base - dot(grad_base, v1v0) * dtt_v0; + const float3 dtt_v1 = (pv0 - 2.0f * tt * v1v0) / t_bot; + grad_v1 = -dot(grad_base, v1v0) * dtt_v1 - tt * grad_base; + } + + return thrust::make_tuple(grad_p, grad_v0, grad_v1); +} + +// Computes the squared distance of a point p relative to a triangle (v0, v1, +// v2). If the point's projection p0 on the plane spanned by (v0, v1, v2) is +// inside the triangle with vertices (v0, v1, v2), then the returned value is +// the squared distance of p to its projection p0. Otherwise, the returned value +// is the smallest squared distance of p from the line segments (v0, v1), (v0, +// v2) and (v1, v2). +// +// Args: +// p: vec3 coordinates of a point +// v0, v1, v2: vec3 coordinates of the triangle vertices +// min_triangle_area: triangles less than this size are considered +// points/lines, IsInsideTriangle returns False +// +// Returns: +// dist: Float of the squared distance +// + +__device__ inline float PointTriangle3DistanceForward( + const float3& p, + const float3& v0, + const float3& v1, + const float3& v2, + const double min_triangle_area) { + float3 normal = cross(v2 - v0, v1 - v0); + const float norm_normal = norm(normal); + normal = normalize(normal); + + // p0 is the projection of p on the plane spanned by (v0, v1, v2) + // i.e. p0 = p + t * normal, s.t. (p0 - v0) is orthogonal to normal + const float t = dot(v0 - p, normal); + const float3 p0 = p + t * normal; + + bool is_inside = IsInsideTriangle(p0, v0, v1, v2, min_triangle_area); + float dist = 0.0f; + + if ((is_inside) && (norm_normal > kEpsilon)) { + // if projection p0 is inside triangle spanned by (v0, v1, v2) + // then distance is equal to norm(p0 - p)^2 + dist = t * t; + } else { + const float e01 = PointLine3DistanceForward(p, v0, v1); + const float e02 = PointLine3DistanceForward(p, v0, v2); + const float e12 = PointLine3DistanceForward(p, v1, v2); + + dist = (e01 > e02) ? e02 : e01; + dist = (dist > e12) ? e12 : dist; + } + + return dist; +} + +// The backward pass for computing the squared distance of a point +// to the triangle (v0, v1, v2). +// +// Args: +// p: xyz coordinates of a point +// v0, v1, v2: xyz coordinates of the triangle vertices +// grad_dist: Float of the gradient wrt dist +// min_triangle_area: triangles less than this size are considered +// points/lines, IsInsideTriangle returns False +// +// Returns: +// tuple of gradients for the point and triangle: +// (float3 grad_p, float3 grad_v0, float3 grad_v1, float3 grad_v2) +// + +__device__ inline thrust::tuple +PointTriangle3DistanceBackward( + const float3& p, + const float3& v0, + const float3& v1, + const float3& v2, + const float& grad_dist, + const double min_triangle_area) { + const float3 v2v0 = v2 - v0; + const float3 v1v0 = v1 - v0; + const float3 v0p = v0 - p; + float3 raw_normal = cross(v2v0, v1v0); + const float norm_normal = norm(raw_normal); + float3 normal = normalize(raw_normal); + + // p0 is the projection of p on the plane spanned by (v0, v1, v2) + // i.e. p0 = p + t * normal, s.t. (p0 - v0) is orthogonal to normal + const float t = dot(v0 - p, normal); + const float3 p0 = p + t * normal; + const float3 diff = t * normal; + + bool is_inside = IsInsideTriangle(p0, v0, v1, v2, min_triangle_area); + + float3 grad_p = make_float3(0.0f, 0.0f, 0.0f); + float3 grad_v0 = make_float3(0.0f, 0.0f, 0.0f); + float3 grad_v1 = make_float3(0.0f, 0.0f, 0.0f); + float3 grad_v2 = make_float3(0.0f, 0.0f, 0.0f); + + if ((is_inside) && (norm_normal > kEpsilon)) { + // derivative of dist wrt p + grad_p = -2.0f * grad_dist * t * normal; + // derivative of dist wrt normal + const float3 grad_normal = 2.0f * grad_dist * t * (v0p + diff); + // derivative of dist wrt raw_normal + const float3 grad_raw_normal = normalize_backward(raw_normal, grad_normal); + // derivative of dist wrt v2v0 and v1v0 + const auto grad_cross = cross_backward(v2v0, v1v0, grad_raw_normal); + const float3 grad_cross_v2v0 = thrust::get<0>(grad_cross); + const float3 grad_cross_v1v0 = thrust::get<1>(grad_cross); + grad_v0 = + grad_dist * 2.0f * t * normal - (grad_cross_v2v0 + grad_cross_v1v0); + grad_v1 = grad_cross_v1v0; + grad_v2 = grad_cross_v2v0; + } else { + const float e01 = PointLine3DistanceForward(p, v0, v1); + const float e02 = PointLine3DistanceForward(p, v0, v2); + const float e12 = PointLine3DistanceForward(p, v1, v2); + + if ((e01 <= e02) && (e01 <= e12)) { + // e01 is smallest + const auto grads = PointLine3DistanceBackward(p, v0, v1, grad_dist); + grad_p = thrust::get<0>(grads); + grad_v0 = thrust::get<1>(grads); + grad_v1 = thrust::get<2>(grads); + } else if ((e02 <= e01) && (e02 <= e12)) { + // e02 is smallest + const auto grads = PointLine3DistanceBackward(p, v0, v2, grad_dist); + grad_p = thrust::get<0>(grads); + grad_v0 = thrust::get<1>(grads); + grad_v2 = thrust::get<2>(grads); + } else if ((e12 <= e01) && (e12 <= e02)) { + // e12 is smallest + const auto grads = PointLine3DistanceBackward(p, v1, v2, grad_dist); + grad_p = thrust::get<0>(grads); + grad_v1 = thrust::get<1>(grads); + grad_v2 = thrust::get<2>(grads); + } + } + + return thrust::make_tuple(grad_p, grad_v0, grad_v1, grad_v2); +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/geometry_utils.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/geometry_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..ad9f7ff3f34dde7b119ea708eb0901cb826794d7 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/geometry_utils.h @@ -0,0 +1,823 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include +#include +#include "vec2.h" +#include "vec3.h" + +// Set epsilon for preventing floating point errors and division by 0. +const auto kEpsilon = 1e-8; + +// Determines whether a point p is on the right side of a 2D line segment +// given by the end points v0, v1. +// +// Args: +// p: vec2 Coordinates of a point. +// v0, v1: vec2 Coordinates of the end points of the edge. +// +// Returns: +// area: The signed area of the parallelogram given by the vectors +// A = p - v0 +// B = v1 - v0 +// +// v1 ________ +// /\ / +// A / \ / +// / \ / +// v0 /______\/ +// B p +// +// The area can also be interpreted as the cross product A x B. +// If the sign of the area is positive, the point p is on the +// right side of the edge. Negative area indicates the point is on +// the left side of the edge. i.e. for an edge v1 - v0: +// +// v1 +// / +// / +// - / + +// / +// / +// v0 +// +template +T EdgeFunctionForward(const vec2& p, const vec2& v0, const vec2& v1) { + const T edge = (p.x - v0.x) * (v1.y - v0.y) - (p.y - v0.y) * (v1.x - v0.x); + return edge; +} + +// Backward pass for the edge function returning partial dervivatives for each +// of the input points. +// +// Args: +// p: vec2 Coordinates of a point. +// v0, v1: vec2 Coordinates of the end points of the edge. +// grad_edge: Upstream gradient for output from edge function. +// +// Returns: +// tuple of gradients for each of the input points: +// (vec2 d_edge_dp, vec2 d_edge_dv0, vec2 d_edge_dv1) +// +template +inline std::tuple, vec2, vec2> EdgeFunctionBackward( + const vec2& p, + const vec2& v0, + const vec2& v1, + const T grad_edge) { + const vec2 dedge_dp(v1.y - v0.y, v0.x - v1.x); + const vec2 dedge_dv0(p.y - v1.y, v1.x - p.x); + const vec2 dedge_dv1(v0.y - p.y, p.x - v0.x); + return std::make_tuple( + grad_edge * dedge_dp, grad_edge * dedge_dv0, grad_edge * dedge_dv1); +} + +// The forward pass for computing the barycentric coordinates of a point +// relative to a triangle. +// Ref: +// https://www.scratchapixel.com/lessons/3d-basic-rendering/ray-tracing-rendering-a-triangle/barycentric-coordinates +// +// Args: +// p: Coordinates of a point. +// v0, v1, v2: Coordinates of the triangle vertices. +// +// Returns +// bary: (w0, w1, w2) barycentric coordinates in the range [0, 1]. +// +template +vec3 BarycentricCoordinatesForward( + const vec2& p, + const vec2& v0, + const vec2& v1, + const vec2& v2) { + const T area = EdgeFunctionForward(v2, v0, v1) + kEpsilon; + const T w0 = EdgeFunctionForward(p, v1, v2) / area; + const T w1 = EdgeFunctionForward(p, v2, v0) / area; + const T w2 = EdgeFunctionForward(p, v0, v1) / area; + return vec3(w0, w1, w2); +} + +// The backward pass for computing the barycentric coordinates of a point +// relative to a triangle. +// +// Args: +// p: Coordinates of a point. +// v0, v1, v2: (x, y) coordinates of the triangle vertices. +// grad_bary_upstream: vec3 Upstream gradient for each of the +// barycentric coordaintes [grad_w0, grad_w1, grad_w2]. +// +// Returns +// tuple of gradients for each of the triangle vertices: +// (vec2 grad_v0, vec2 grad_v1, vec2 grad_v2) +// +template +inline std::tuple, vec2, vec2, vec2> BarycentricCoordsBackward( + const vec2& p, + const vec2& v0, + const vec2& v1, + const vec2& v2, + const vec3& grad_bary_upstream) { + const T area = EdgeFunctionForward(v2, v0, v1) + kEpsilon; + const T area2 = pow(area, 2.0f); + const T area_inv = 1.0f / area; + const T e0 = EdgeFunctionForward(p, v1, v2); + const T e1 = EdgeFunctionForward(p, v2, v0); + const T e2 = EdgeFunctionForward(p, v0, v1); + + const T grad_w0 = grad_bary_upstream.x; + const T grad_w1 = grad_bary_upstream.y; + const T grad_w2 = grad_bary_upstream.z; + + // Calculate component of the gradient from each of w0, w1 and w2. + // e.g. for w0: + // dloss/dw0_v = dl/dw0 * dw0/dw0_top * dw0_top/dv + // + dl/dw0 * dw0/dw0_bot * dw0_bot/dv + const T dw0_darea = -e0 / (area2); + const T dw0_e0 = area_inv; + const T dloss_d_w0area = grad_w0 * dw0_darea; + const T dloss_e0 = grad_w0 * dw0_e0; + auto de0_dv = EdgeFunctionBackward(p, v1, v2, dloss_e0); + auto dw0area_dv = EdgeFunctionBackward(v2, v0, v1, dloss_d_w0area); + const vec2 dw0_p = std::get<0>(de0_dv); + const vec2 dw0_dv0 = std::get<1>(dw0area_dv); + const vec2 dw0_dv1 = std::get<1>(de0_dv) + std::get<2>(dw0area_dv); + const vec2 dw0_dv2 = std::get<2>(de0_dv) + std::get<0>(dw0area_dv); + + const T dw1_darea = -e1 / (area2); + const T dw1_e1 = area_inv; + const T dloss_d_w1area = grad_w1 * dw1_darea; + const T dloss_e1 = grad_w1 * dw1_e1; + auto de1_dv = EdgeFunctionBackward(p, v2, v0, dloss_e1); + auto dw1area_dv = EdgeFunctionBackward(v2, v0, v1, dloss_d_w1area); + const vec2 dw1_p = std::get<0>(de1_dv); + const vec2 dw1_dv0 = std::get<2>(de1_dv) + std::get<1>(dw1area_dv); + const vec2 dw1_dv1 = std::get<2>(dw1area_dv); + const vec2 dw1_dv2 = std::get<1>(de1_dv) + std::get<0>(dw1area_dv); + + const T dw2_darea = -e2 / (area2); + const T dw2_e2 = area_inv; + const T dloss_d_w2area = grad_w2 * dw2_darea; + const T dloss_e2 = grad_w2 * dw2_e2; + auto de2_dv = EdgeFunctionBackward(p, v0, v1, dloss_e2); + auto dw2area_dv = EdgeFunctionBackward(v2, v0, v1, dloss_d_w2area); + const vec2 dw2_p = std::get<0>(de2_dv); + const vec2 dw2_dv0 = std::get<1>(de2_dv) + std::get<1>(dw2area_dv); + const vec2 dw2_dv1 = std::get<2>(de2_dv) + std::get<2>(dw2area_dv); + const vec2 dw2_dv2 = std::get<0>(dw2area_dv); + + const vec2 dbary_p = dw0_p + dw1_p + dw2_p; + const vec2 dbary_dv0 = dw0_dv0 + dw1_dv0 + dw2_dv0; + const vec2 dbary_dv1 = dw0_dv1 + dw1_dv1 + dw2_dv1; + const vec2 dbary_dv2 = dw0_dv2 + dw1_dv2 + dw2_dv2; + + return std::make_tuple(dbary_p, dbary_dv0, dbary_dv1, dbary_dv2); +} + +// Forward pass for applying perspective correction to barycentric coordinates. +// +// Args: +// bary: Screen-space barycentric coordinates for a point +// z0, z1, z2: Camera-space z-coordinates of the triangle vertices +// +// Returns +// World-space barycentric coordinates +// +template +inline vec3 BarycentricPerspectiveCorrectionForward( + const vec3& bary, + const T z0, + const T z1, + const T z2) { + const T w0_top = bary.x * z1 * z2; + const T w1_top = bary.y * z0 * z2; + const T w2_top = bary.z * z0 * z1; + const T denom = std::max(w0_top + w1_top + w2_top, kEpsilon); + const T w0 = w0_top / denom; + const T w1 = w1_top / denom; + const T w2 = w2_top / denom; + return vec3(w0, w1, w2); +} + +// Backward pass for applying perspective correction to barycentric coordinates. +// +// Args: +// bary: Screen-space barycentric coordinates for a point +// z0, z1, z2: Camera-space z-coordinates of the triangle vertices +// grad_out: Upstream gradient of the loss with respect to the corrected +// barycentric coordinates. +// +// Returns a tuple of: +// grad_bary: Downstream gradient of the loss with respect to the the +// uncorrected barycentric coordinates. +// grad_z0, grad_z1, grad_z2: Downstream gradient of the loss with respect +// to the z-coordinates of the triangle verts +template +inline std::tuple, T, T, T> BarycentricPerspectiveCorrectionBackward( + const vec3& bary, + const T z0, + const T z1, + const T z2, + const vec3& grad_out) { + // Recompute forward pass + const T w0_top = bary.x * z1 * z2; + const T w1_top = bary.y * z0 * z2; + const T w2_top = bary.z * z0 * z1; + const T denom = std::max(w0_top + w1_top + w2_top, kEpsilon); + + // Now do backward pass + const T grad_denom_top = + -w0_top * grad_out.x - w1_top * grad_out.y - w2_top * grad_out.z; + const T grad_denom = grad_denom_top / (denom * denom); + const T grad_w0_top = grad_denom + grad_out.x / denom; + const T grad_w1_top = grad_denom + grad_out.y / denom; + const T grad_w2_top = grad_denom + grad_out.z / denom; + const T grad_bary_x = grad_w0_top * z1 * z2; + const T grad_bary_y = grad_w1_top * z0 * z2; + const T grad_bary_z = grad_w2_top * z0 * z1; + const vec3 grad_bary(grad_bary_x, grad_bary_y, grad_bary_z); + const T grad_z0 = grad_w1_top * bary.y * z2 + grad_w2_top * bary.z * z1; + const T grad_z1 = grad_w0_top * bary.x * z2 + grad_w2_top * bary.z * z0; + const T grad_z2 = grad_w0_top * bary.x * z1 + grad_w1_top * bary.y * z0; + return std::make_tuple(grad_bary, grad_z0, grad_z1, grad_z2); +} + +// Clip negative barycentric coordinates to 0.0 and renormalize so +// the barycentric coordinates for a point sum to 1. When the blur_radius +// is greater than 0, a face will still be recorded as overlapping a pixel +// if the pixel is outside the face. In this case at least one of the +// barycentric coordinates for the pixel relative to the face will be negative. +// Clipping will ensure that the texture and z buffer are interpolated +// correctly. +// +// Args +// bary: (w0, w1, w2) barycentric coordinates which can contain values < 0. +// +// Returns +// bary: (w0, w1, w2) barycentric coordinates in the range [0, 1] which +// satisfy the condition: sum(w0, w1, w2) = 1.0. +// +template +vec3 BarycentricClipForward(const vec3 bary) { + vec3 w(0.0f, 0.0f, 0.0f); + // Only clamp negative values to 0.0. + // No need to clamp values > 1.0 as they will be renormalized. + w.x = std::max(bary.x, 0.0f); + w.y = std::max(bary.y, 0.0f); + w.z = std::max(bary.z, 0.0f); + float w_sum = w.x + w.y + w.z; + w_sum = std::fmaxf(w_sum, 1e-5); + w.x /= w_sum; + w.y /= w_sum; + w.z /= w_sum; + return w; +} + +// Backward pass for barycentric coordinate clipping. +// +// Args +// bary: (w0, w1, w2) barycentric coordinates which can contain values < 0. +// grad_baryclip_upstream: vec3 Upstream gradient for each of the clipped +// barycentric coordinates [grad_w0, grad_w1, grad_w2]. +// +// Returns +// vec3 of gradients for the unclipped barycentric coordinates: +// (grad_w0, grad_w1, grad_w2) +// +template +vec3 BarycentricClipBackward( + const vec3 bary, + const vec3 grad_baryclip_upstream) { + // Redo some of the forward pass calculations + vec3 w(0.0f, 0.0f, 0.0f); + w.x = std::max(bary.x, 0.0f); + w.y = std::max(bary.y, 0.0f); + w.z = std::max(bary.z, 0.0f); + float w_sum = w.x + w.y + w.z; + + vec3 grad_bary(1.0f, 1.0f, 1.0f); + vec3 grad_clip(1.0f, 1.0f, 1.0f); + vec3 grad_sum(1.0f, 1.0f, 1.0f); + + // Check if the sum was clipped. + float grad_sum_clip = 1.0f; + if (w_sum < 1e-5) { + grad_sum_clip = 0.0f; + w_sum = 1e-5; + } + + // Check if any of the bary coordinates have been clipped. + // Only negative values are clamped to 0.0. + if (bary.x < 0.0f) { + grad_clip.x = 0.0f; + } + if (bary.y < 0.0f) { + grad_clip.y = 0.0f; + } + if (bary.z < 0.0f) { + grad_clip.z = 0.0f; + } + + // Gradients of the sum. + grad_sum.x = -w.x / (pow(w_sum, 2.0f)) * grad_sum_clip; + grad_sum.y = -w.y / (pow(w_sum, 2.0f)) * grad_sum_clip; + grad_sum.z = -w.z / (pow(w_sum, 2.0f)) * grad_sum_clip; + + // Gradients for each of the bary coordinates including the cross terms + // from the sum. + grad_bary.x = grad_clip.x * + (grad_baryclip_upstream.x * (1.0f / w_sum + grad_sum.x) + + grad_baryclip_upstream.y * (grad_sum.y) + + grad_baryclip_upstream.z * (grad_sum.z)); + + grad_bary.y = grad_clip.y * + (grad_baryclip_upstream.y * (1.0f / w_sum + grad_sum.y) + + grad_baryclip_upstream.x * (grad_sum.x) + + grad_baryclip_upstream.z * (grad_sum.z)); + + grad_bary.z = grad_clip.z * + (grad_baryclip_upstream.z * (1.0f / w_sum + grad_sum.z) + + grad_baryclip_upstream.x * (grad_sum.x) + + grad_baryclip_upstream.y * (grad_sum.y)); + + return grad_bary; +} + +// Calculate minimum distance between a line segment (v1 - v0) and point p. +// +// Args: +// p: Coordinates of a point. +// v0, v1: Coordinates of the end points of the line segment. +// +// Returns: +// squared distance of the point to the line. +// +// Consider the line extending the segment - this can be parameterized as: +// v0 + t (v1 - v0). +// +// First find the projection of point p onto the line. It falls where: +// t = [(p - v0) . (v1 - v0)] / |v1 - v0|^2 +// where . is the dot product. +// +// The parameter t is clamped from [0, 1] to handle points outside the +// segment (v1 - v0). +// +// Once the projection of the point on the segment is known, the distance from +// p to the projection gives the minimum distance to the segment. +// +template +T PointLineDistanceForward( + const vec2& p, + const vec2& v0, + const vec2& v1) { + const vec2 v1v0 = v1 - v0; + const T l2 = dot(v1v0, v1v0); + if (l2 <= kEpsilon) { + return dot(p - v1, p - v1); + } + + const T t = dot(v1v0, p - v0) / l2; + const T tt = std::min(std::max(t, 0.00f), 1.00f); + const vec2 p_proj = v0 + tt * v1v0; + return dot(p - p_proj, p - p_proj); +} + +template +T PointLine3DistanceForward( + const vec3& p, + const vec3& v0, + const vec3& v1) { + const vec3 v1v0 = v1 - v0; + const T l2 = dot(v1v0, v1v0); + if (l2 <= kEpsilon) { + return dot(p - v1, p - v1); + } + + const T t = dot(v1v0, p - v0) / l2; + const T tt = std::min(std::max(t, 0.00f), 1.00f); + const vec3 p_proj = v0 + tt * v1v0; + return dot(p - p_proj, p - p_proj); +} + +// Backward pass for point to line distance in 2D. +// +// Args: +// p: Coordinates of a point. +// v0, v1: Coordinates of the end points of the line segment. +// grad_dist: Upstream gradient for the distance. +// +// Returns: +// tuple of gradients for each of the input points: +// (vec2 grad_p, vec2 grad_v0, vec2 grad_v1) +// +template +inline std::tuple, vec2, vec2> PointLineDistanceBackward( + const vec2& p, + const vec2& v0, + const vec2& v1, + const T& grad_dist) { + // Redo some of the forward pass calculations. + const vec2 v1v0 = v1 - v0; + const vec2 pv0 = p - v0; + const T t_bot = dot(v1v0, v1v0); + const T t_top = dot(v1v0, pv0); + const T t = t_top / t_bot; + const T tt = std::min(std::max(t, 0.00f), 1.00f); + const vec2 p_proj = (1.0f - tt) * v0 + tt * v1; + + const vec2 grad_v0 = grad_dist * (1.0f - tt) * 2.0f * (p_proj - p); + const vec2 grad_v1 = grad_dist * tt * 2.0f * (p_proj - p); + const vec2 grad_p = -1.0f * grad_dist * 2.0f * (p_proj - p); + + return std::make_tuple(grad_p, grad_v0, grad_v1); +} + +template +std::tuple, vec3, vec3> PointLine3DistanceBackward( + const vec3& p, + const vec3& v0, + const vec3& v1, + const T& grad_dist) { + const vec3 v1v0 = v1 - v0; + const vec3 pv0 = p - v0; + const T t_bot = dot(v1v0, v1v0); + const T t_top = dot(v1v0, pv0); + + vec3 grad_p{0.0f, 0.0f, 0.0f}; + vec3 grad_v0{0.0f, 0.0f, 0.0f}; + vec3 grad_v1{0.0f, 0.0f, 0.0f}; + + const T tt = t_top / t_bot; + + if (t_bot < kEpsilon) { + // if t_bot small, then v0 == v1, + // and dist = 0.5 * dot(pv0, pv0) + 0.5 * dot(pv1, pv1) + grad_p = grad_dist * 2.0f * pv0; + grad_v0 = -0.5f * grad_p; + grad_v1 = grad_v0; + } else if (tt < 0.0f) { + grad_p = grad_dist * 2.0f * pv0; + grad_v0 = -1.0f * grad_p; + // no gradients wrt v1 + } else if (tt > 1.0f) { + grad_p = grad_dist * 2.0f * (p - v1); + grad_v1 = -1.0f * grad_p; + // no gradients wrt v0 + } else { + const vec3 p_proj = v0 + tt * v1v0; + const vec3 diff = p - p_proj; + const vec3 grad_base = grad_dist * 2.0f * diff; + grad_p = grad_base - dot(grad_base, v1v0) * v1v0 / t_bot; + const vec3 dtt_v0 = (-1.0f * v1v0 - pv0 + 2.0f * tt * v1v0) / t_bot; + grad_v0 = (-1.0f + tt) * grad_base - dot(grad_base, v1v0) * dtt_v0; + const vec3 dtt_v1 = (pv0 - 2.0f * tt * v1v0) / t_bot; + grad_v1 = -dot(grad_base, v1v0) * dtt_v1 - tt * grad_base; + } + + return std::make_tuple(grad_p, grad_v0, grad_v1); +} + +// The forward pass for calculating the shortest distance between a point +// and a triangle. +// Ref: https://www.randygaul.net/2014/07/23/distance-point-to-line-segment/ +// +// Args: +// p: Coordinates of a point. +// v0, v1, v2: Coordinates of the three triangle vertices. +// +// Returns: +// shortest squared distance from a point to a triangle. +// +// +template +T PointTriangleDistanceForward( + const vec2& p, + const vec2& v0, + const vec2& v1, + const vec2& v2) { + // Compute distance of point to 3 edges of the triangle and return the + // minimum value. + const T e01_dist = PointLineDistanceForward(p, v0, v1); + const T e02_dist = PointLineDistanceForward(p, v0, v2); + const T e12_dist = PointLineDistanceForward(p, v1, v2); + const T edge_dist = std::min(std::min(e01_dist, e02_dist), e12_dist); + + return edge_dist; +} + +// Backward pass for point triangle distance. +// +// Args: +// p: Coordinates of a point. +// v0, v1, v2: Coordinates of the three triangle vertices. +// grad_dist: Upstream gradient for the distance. +// +// Returns: +// tuple of gradients for each of the triangle vertices: +// (vec2 grad_v0, vec2 grad_v1, vec2 grad_v2) +// +template +inline std::tuple, vec2, vec2, vec2> +PointTriangleDistanceBackward( + const vec2& p, + const vec2& v0, + const vec2& v1, + const vec2& v2, + const T& grad_dist) { + // Compute distance to all 3 edges of the triangle. + const T e01_dist = PointLineDistanceForward(p, v0, v1); + const T e02_dist = PointLineDistanceForward(p, v0, v2); + const T e12_dist = PointLineDistanceForward(p, v1, v2); + + // Initialize output tensors. + vec2 grad_v0(0.0f, 0.0f); + vec2 grad_v1(0.0f, 0.0f); + vec2 grad_v2(0.0f, 0.0f); + vec2 grad_p(0.0f, 0.0f); + + // Find which edge is the closest and return PointLineDistanceBackward for + // that edge. + if (e01_dist <= e02_dist && e01_dist <= e12_dist) { + // Closest edge is v1 - v0. + auto grad_e01 = PointLineDistanceBackward(p, v0, v1, grad_dist); + grad_p = std::get<0>(grad_e01); + grad_v0 = std::get<1>(grad_e01); + grad_v1 = std::get<2>(grad_e01); + } else if (e02_dist <= e01_dist && e02_dist <= e12_dist) { + // Closest edge is v2 - v0. + auto grad_e02 = PointLineDistanceBackward(p, v0, v2, grad_dist); + grad_p = std::get<0>(grad_e02); + grad_v0 = std::get<1>(grad_e02); + grad_v2 = std::get<2>(grad_e02); + } else if (e12_dist <= e01_dist && e12_dist <= e02_dist) { + // Closest edge is v2 - v1. + auto grad_e12 = PointLineDistanceBackward(p, v1, v2, grad_dist); + grad_p = std::get<0>(grad_e12); + grad_v1 = std::get<1>(grad_e12); + grad_v2 = std::get<2>(grad_e12); + } + + return std::make_tuple(grad_p, grad_v0, grad_v1, grad_v2); +} + +// Computes the area of a triangle (v0, v1, v2). +// Args: +// v0, v1, v2: vec3 coordinates of the triangle vertices +// +// Returns: +// area: float: the area of the triangle +// +template +T AreaOfTriangle(const vec3& v0, const vec3& v1, const vec3& v2) { + vec3 p0 = v1 - v0; + vec3 p1 = v2 - v0; + + // compute the hypotenus of the scross product (p0 x p1) + float dd = std::hypot( + p0.y * p1.z - p0.z * p1.y, + std::hypot(p0.z * p1.x - p0.x * p1.z, p0.x * p1.y - p0.y * p1.x)); + + return dd / 2.0; +} + +// Computes the squared distance of a point p relative to a triangle (v0, v1, +// v2). If the point's projection p0 on the plane spanned by (v0, v1, v2) is +// inside the triangle with vertices (v0, v1, v2), then the returned value is +// the squared distance of p to its projection p0. Otherwise, the returned value +// is the smallest squared distance of p from the line segments (v0, v1), (v0, +// v2) and (v1, v2). +// +// Args: +// p: vec3 coordinates of a point +// v0, v1, v2: vec3 coordinates of the triangle vertices +// +// Returns: +// dist: Float of the squared distance +// + +const float vEpsilon = 1e-8; + +template +vec3 BarycentricCoords3Forward( + const vec3& p, + const vec3& v0, + const vec3& v1, + const vec3& v2) { + vec3 p0 = v1 - v0; + vec3 p1 = v2 - v0; + vec3 p2 = p - v0; + + const T d00 = dot(p0, p0); + const T d01 = dot(p0, p1); + const T d11 = dot(p1, p1); + const T d20 = dot(p2, p0); + const T d21 = dot(p2, p1); + + const T denom = d00 * d11 - d01 * d01 + kEpsilon; + const T w1 = (d11 * d20 - d01 * d21) / denom; + const T w2 = (d00 * d21 - d01 * d20) / denom; + const T w0 = 1.0f - w1 - w2; + + return vec3(w0, w1, w2); +} + +// Checks whether the point p is inside the triangle (v0, v1, v2). +// A point is inside the triangle, if all barycentric coordinates +// wrt the triangle are >= 0 & <= 1. +// If the triangle is degenerate, aka line or point, then return False. +// +// NOTE that this function assumes that p lives on the space spanned +// by (v0, v1, v2). +// TODO(gkioxari) explicitly check whether p is coplanar with (v0, v1, v2) +// and throw an error if check fails +// +// Args: +// p: vec3 coordinates of a point +// v0, v1, v2: vec3 coordinates of the triangle vertices +// min_triangle_area: triangles less than this size are considered +// points/lines, IsInsideTriangle returns False +// +// Returns: +// inside: bool indicating wether p is inside triangle +// +template +static bool IsInsideTriangle( + const vec3& p, + const vec3& v0, + const vec3& v1, + const vec3& v2, + const double min_triangle_area) { + bool inside; + if (AreaOfTriangle(v0, v1, v2) < min_triangle_area) { + inside = 0; + } else { + vec3 bary = BarycentricCoords3Forward(p, v0, v1, v2); + bool x_in = 0.0f <= bary.x && bary.x <= 1.0f; + bool y_in = 0.0f <= bary.y && bary.y <= 1.0f; + bool z_in = 0.0f <= bary.z && bary.z <= 1.0f; + inside = x_in && y_in && z_in; + } + return inside; +} + +template +T PointTriangle3DistanceForward( + const vec3& p, + const vec3& v0, + const vec3& v1, + const vec3& v2, + const double min_triangle_area) { + vec3 normal = cross(v2 - v0, v1 - v0); + const T norm_normal = norm(normal); + normal = normal / (norm_normal + vEpsilon); + + // p0 is the projection of p on the plane spanned by (v0, v1, v2) + // i.e. p0 = p + t * normal, s.t. (p0 - v0) is orthogonal to normal + const T t = dot(v0 - p, normal); + const vec3 p0 = p + t * normal; + + bool is_inside = IsInsideTriangle(p0, v0, v1, v2, min_triangle_area); + T dist = 0.0f; + + if ((is_inside) && (norm_normal > kEpsilon)) { + // if projection p0 is inside triangle spanned by (v0, v1, v2) + // then distance is equal to norm(p0 - p)^2 + dist = t * t; + } else { + const float e01 = PointLine3DistanceForward(p, v0, v1); + const float e02 = PointLine3DistanceForward(p, v0, v2); + const float e12 = PointLine3DistanceForward(p, v1, v2); + + dist = (e01 > e02) ? e02 : e01; + dist = (dist > e12) ? e12 : dist; + } + + return dist; +} + +template +std::tuple, vec3> +cross_backward(const vec3& a, const vec3& b, const vec3& grad_cross) { + const float grad_ax = -grad_cross.y * b.z + grad_cross.z * b.y; + const float grad_ay = grad_cross.x * b.z - grad_cross.z * b.x; + const float grad_az = -grad_cross.x * b.y + grad_cross.y * b.x; + const vec3 grad_a = vec3(grad_ax, grad_ay, grad_az); + + const float grad_bx = grad_cross.y * a.z - grad_cross.z * a.y; + const float grad_by = -grad_cross.x * a.z + grad_cross.z * a.x; + const float grad_bz = grad_cross.x * a.y - grad_cross.y * a.x; + const vec3 grad_b = vec3(grad_bx, grad_by, grad_bz); + + return std::make_tuple(grad_a, grad_b); +} + +template +vec3 normalize_backward(const vec3& a, const vec3& grad_normz) { + const float a_norm = norm(a) + vEpsilon; + const vec3 out = a / a_norm; + + const float grad_ax = grad_normz.x * (1.0f - out.x * out.x) / a_norm + + grad_normz.y * (-out.x * out.y) / a_norm + + grad_normz.z * (-out.x * out.z) / a_norm; + const float grad_ay = grad_normz.x * (-out.x * out.y) / a_norm + + grad_normz.y * (1.0f - out.y * out.y) / a_norm + + grad_normz.z * (-out.y * out.z) / a_norm; + const float grad_az = grad_normz.x * (-out.x * out.z) / a_norm + + grad_normz.y * (-out.y * out.z) / a_norm + + grad_normz.z * (1.0f - out.z * out.z) / a_norm; + return vec3(grad_ax, grad_ay, grad_az); +} + +// The backward pass for computing the squared distance of a point +// to the triangle (v0, v1, v2). +// +// Args: +// p: xyz coordinates of a point +// v0, v1, v2: xyz coordinates of the triangle vertices +// grad_dist: Float of the gradient wrt dist +// min_triangle_area: triangles less than this size are considered +// points/lines, IsInsideTriangle returns False +// +// Returns: +// tuple of gradients for the point and triangle: +// (float3 grad_p, float3 grad_v0, float3 grad_v1, float3 grad_v2) +// + +template +static std::tuple, vec3, vec3, vec3> +PointTriangle3DistanceBackward( + const vec3& p, + const vec3& v0, + const vec3& v1, + const vec3& v2, + const T& grad_dist, + const double min_triangle_area) { + const vec3 v2v0 = v2 - v0; + const vec3 v1v0 = v1 - v0; + const vec3 v0p = v0 - p; + vec3 raw_normal = cross(v2v0, v1v0); + const T norm_normal = norm(raw_normal); + vec3 normal = raw_normal / (norm_normal + vEpsilon); + + // p0 is the projection of p on the plane spanned by (v0, v1, v2) + // i.e. p0 = p + t * normal, s.t. (p0 - v0) is orthogonal to normal + const T t = dot(v0 - p, normal); + const vec3 p0 = p + t * normal; + const vec3 diff = t * normal; + + bool is_inside = IsInsideTriangle(p0, v0, v1, v2, min_triangle_area); + + vec3 grad_p(0.0f, 0.0f, 0.0f); + vec3 grad_v0(0.0f, 0.0f, 0.0f); + vec3 grad_v1(0.0f, 0.0f, 0.0f); + vec3 grad_v2(0.0f, 0.0f, 0.0f); + + if ((is_inside) && (norm_normal > kEpsilon)) { + // derivative of dist wrt p + grad_p = -2.0f * grad_dist * t * normal; + // derivative of dist wrt normal + const vec3 grad_normal = 2.0f * grad_dist * t * (v0p + diff); + // derivative of dist wrt raw_normal + const vec3 grad_raw_normal = normalize_backward(raw_normal, grad_normal); + // derivative of dist wrt v2v0 and v1v0 + const auto grad_cross = cross_backward(v2v0, v1v0, grad_raw_normal); + const vec3 grad_cross_v2v0 = std::get<0>(grad_cross); + const vec3 grad_cross_v1v0 = std::get<1>(grad_cross); + grad_v0 = + grad_dist * 2.0f * t * normal - (grad_cross_v2v0 + grad_cross_v1v0); + grad_v1 = grad_cross_v1v0; + grad_v2 = grad_cross_v2v0; + } else { + const T e01 = PointLine3DistanceForward(p, v0, v1); + const T e02 = PointLine3DistanceForward(p, v0, v2); + const T e12 = PointLine3DistanceForward(p, v1, v2); + + if ((e01 <= e02) && (e01 <= e12)) { + // e01 is smallest + const auto grads = PointLine3DistanceBackward(p, v0, v1, grad_dist); + grad_p = std::get<0>(grads); + grad_v0 = std::get<1>(grads); + grad_v1 = std::get<2>(grads); + } else if ((e02 <= e01) && (e02 <= e12)) { + // e02 is smallest + const auto grads = PointLine3DistanceBackward(p, v0, v2, grad_dist); + grad_p = std::get<0>(grads); + grad_v0 = std::get<1>(grads); + grad_v2 = std::get<2>(grads); + } else if ((e12 <= e01) && (e12 <= e02)) { + // e12 is smallest + const auto grads = PointLine3DistanceBackward(p, v1, v2, grad_dist); + grad_p = std::get<0>(grads); + grad_v1 = std::get<1>(grads); + grad_v2 = std::get<2>(grads); + } + } + + return std::make_tuple(grad_p, grad_v0, grad_v1, grad_v2); +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/index_utils.cuh b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/index_utils.cuh new file mode 100644 index 0000000000000000000000000000000000000000..74bca270f344b0d0b0bc19c45fc7373eb5875a14 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/index_utils.cuh @@ -0,0 +1,224 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +// This converts dynamic array lookups into static array lookups, for small +// arrays up to size 32. +// +// Suppose we have a small thread-local array: +// +// float vals[10]; +// +// Ideally we should only index this array using static indices: +// +// for (int i = 0; i < 10; ++i) vals[i] = i * i; +// +// If we do so, then the CUDA compiler may be able to place the array into +// registers, which can have a big performance improvement. However if we +// access the array dynamically, the the compiler may force the array into +// local memory, which has the same latency as global memory. +// +// These functions convert dynamic array access into static array access +// using a brute-force lookup table. It can be used like this: +// +// float vals[10]; +// int idx = 3; +// float val = 3.14f; +// RegisterIndexUtils::set(vals, idx, val); +// float val2 = RegisterIndexUtils::get(vals, idx); +// +// The implementation is based on fbcuda/RegisterUtils.cuh: +// https://github.com/facebook/fbcuda/blob/master/RegisterUtils.cuh +// To avoid depending on the entire library, we just reimplement these two +// functions. The fbcuda implementation is a bit more sophisticated, and uses +// the preprocessor to generate switch statements that go up to N for each +// value of N. We are lazy and just have a giant explicit switch statement. +// +// We might be able to use a template metaprogramming approach similar to +// DispatchKernel1D for this. However DispatchKernel1D is intended to be used +// for dispatching to the correct CUDA kernel on the host, while this is +// is intended to run on the device. I was concerned that a metaprogramming +// approach for this might lead to extra function calls at runtime if the +// compiler fails to optimize them away, which could be very slow on device. +// However I didn't actually benchmark or test this. +template +struct RegisterIndexUtils { + __device__ __forceinline__ static T get(const T arr[N], int idx) { + if (idx < 0 || idx >= N) + return T(); + switch (idx) { + case 0: + return arr[0]; + case 1: + return arr[1]; + case 2: + return arr[2]; + case 3: + return arr[3]; + case 4: + return arr[4]; + case 5: + return arr[5]; + case 6: + return arr[6]; + case 7: + return arr[7]; + case 8: + return arr[8]; + case 9: + return arr[9]; + case 10: + return arr[10]; + case 11: + return arr[11]; + case 12: + return arr[12]; + case 13: + return arr[13]; + case 14: + return arr[14]; + case 15: + return arr[15]; + case 16: + return arr[16]; + case 17: + return arr[17]; + case 18: + return arr[18]; + case 19: + return arr[19]; + case 20: + return arr[20]; + case 21: + return arr[21]; + case 22: + return arr[22]; + case 23: + return arr[23]; + case 24: + return arr[24]; + case 25: + return arr[25]; + case 26: + return arr[26]; + case 27: + return arr[27]; + case 28: + return arr[28]; + case 29: + return arr[29]; + case 30: + return arr[30]; + case 31: + return arr[31]; + }; + return T(); + } + + __device__ __forceinline__ static void set(T arr[N], int idx, T val) { + if (idx < 0 || idx >= N) + return; + switch (idx) { + case 0: + arr[0] = val; + break; + case 1: + arr[1] = val; + break; + case 2: + arr[2] = val; + break; + case 3: + arr[3] = val; + break; + case 4: + arr[4] = val; + break; + case 5: + arr[5] = val; + break; + case 6: + arr[6] = val; + break; + case 7: + arr[7] = val; + break; + case 8: + arr[8] = val; + break; + case 9: + arr[9] = val; + break; + case 10: + arr[10] = val; + break; + case 11: + arr[11] = val; + break; + case 12: + arr[12] = val; + break; + case 13: + arr[13] = val; + break; + case 14: + arr[14] = val; + break; + case 15: + arr[15] = val; + break; + case 16: + arr[16] = val; + break; + case 17: + arr[17] = val; + break; + case 18: + arr[18] = val; + break; + case 19: + arr[19] = val; + break; + case 20: + arr[20] = val; + break; + case 21: + arr[21] = val; + break; + case 22: + arr[22] = val; + break; + case 23: + arr[23] = val; + break; + case 24: + arr[24] = val; + break; + case 25: + arr[25] = val; + break; + case 26: + arr[26] = val; + break; + case 27: + arr[27] = val; + break; + case 28: + arr[28] = val; + break; + case 29: + arr[29] = val; + break; + case 30: + arr[30] = val; + break; + case 31: + arr[31] = val; + break; + } + } +}; diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/mink.cuh b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/mink.cuh new file mode 100644 index 0000000000000000000000000000000000000000..c7858f58c8b92c12f5e889c10fe6e98a622d82b7 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/mink.cuh @@ -0,0 +1,165 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once +#define MINK_H + +#include "index_utils.cuh" + +// A data structure to keep track of the smallest K keys seen so far as well +// as their associated values, intended to be used in device code. +// This data structure doesn't allocate any memory; keys and values are stored +// in arrays passed to the constructor. +// +// The implementation is generic; it can be used for any key type that supports +// the < operator, and can be used with any value type. +// +// Example usage: +// +// float keys[K]; +// int values[K]; +// MinK mink(keys, values, K); +// for (...) { +// // Produce some key and value from somewhere +// mink.add(key, value); +// } +// mink.sort(); +// +// Now keys and values store the smallest K keys seen so far and the values +// associated to these keys: +// +// for (int k = 0; k < K; ++k) { +// float key_k = keys[k]; +// int value_k = values[k]; +// } +template +class MinK { + public: + // Constructor. + // + // Arguments: + // keys: Array in which to store keys + // values: Array in which to store values + // K: How many values to keep track of + __device__ MinK(key_t* keys, value_t* vals, int K) + : keys(keys), vals(vals), K(K), _size(0) {} + + // Try to add a new key and associated value to the data structure. If the key + // is one of the smallest K seen so far then it will be kept; otherwise it + // it will not be kept. + // + // This takes O(1) operations if the new key is not kept, or if the structure + // currently contains fewer than K elements. Otherwise this takes O(K) time. + // + // Arguments: + // key: The key to add + // val: The value associated to the key + __device__ __forceinline__ void add(const key_t& key, const value_t& val) { + if (_size < K) { + keys[_size] = key; + vals[_size] = val; + if (_size == 0 || key > max_key) { + max_key = key; + max_idx = _size; + } + _size++; + } else if (key < max_key) { + keys[max_idx] = key; + vals[max_idx] = val; + max_key = key; + for (int k = 0; k < K; ++k) { + key_t cur_key = keys[k]; + if (cur_key > max_key) { + max_key = cur_key; + max_idx = k; + } + } + } + } + + // Get the number of items currently stored in the structure. + // This takes O(1) time. + __device__ __forceinline__ int size() { + return _size; + } + + // Sort the items stored in the structure using bubble sort. + // This takes O(K^2) time. + __device__ __forceinline__ void sort() { + for (int i = 0; i < _size - 1; ++i) { + for (int j = 0; j < _size - i - 1; ++j) { + if (keys[j + 1] < keys[j]) { + key_t key = keys[j]; + value_t val = vals[j]; + keys[j] = keys[j + 1]; + vals[j] = vals[j + 1]; + keys[j + 1] = key; + vals[j + 1] = val; + } + } + } + } + + private: + key_t* keys; + value_t* vals; + int K; + int _size; + key_t max_key; + int max_idx; +}; + +// This is a version of MinK that only touches the arrays using static indexing +// via RegisterIndexUtils. If the keys and values are stored in thread-local +// arrays, then this may allow the compiler to place them in registers for +// fast access. +// +// This has the same API as RegisterMinK, but doesn't support sorting. +// We found that sorting via RegisterIndexUtils gave very poor performance, +// and suspect it may have prevented the compiler from placing the arrays +// into registers. +template +class RegisterMinK { + public: + __device__ RegisterMinK(key_t* keys, value_t* vals) + : keys(keys), vals(vals), _size(0) {} + + __device__ __forceinline__ void add(const key_t& key, const value_t& val) { + if (_size < K) { + RegisterIndexUtils::set(keys, _size, key); + RegisterIndexUtils::set(vals, _size, val); + if (_size == 0 || key > max_key) { + max_key = key; + max_idx = _size; + } + _size++; + } else if (key < max_key) { + RegisterIndexUtils::set(keys, max_idx, key); + RegisterIndexUtils::set(vals, max_idx, val); + max_key = key; + for (int k = 0; k < K; ++k) { + key_t cur_key = RegisterIndexUtils::get(keys, k); + if (cur_key > max_key) { + max_key = cur_key; + max_idx = k; + } + } + } + } + + __device__ __forceinline__ int size() { + return _size; + } + + private: + key_t* keys; + value_t* vals; + int _size; + key_t max_key; + int max_idx; +}; diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/pytorch3d_cutils.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/pytorch3d_cutils.h new file mode 100644 index 0000000000000000000000000000000000000000..48d04546e6f96fd7eb2c182f7b6d692f5821c1d8 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/pytorch3d_cutils.h @@ -0,0 +1,17 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once +#include + +#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor.") +#define CHECK_CONTIGUOUS(x) \ + TORCH_CHECK(x.is_contiguous(), #x " must be contiguous.") +#define CHECK_CONTIGUOUS_CUDA(x) \ + CHECK_CUDA(x); \ + CHECK_CONTIGUOUS(x) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/vec2.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/vec2.h new file mode 100644 index 0000000000000000000000000000000000000000..f4550f918394c06ecbcca5db013e4f0f014ee914 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/vec2.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once +#include + +// A fixed-sized vector with basic arithmetic operators useful for +// representing 2D coordinates. +// TODO: switch to Eigen if more functionality is needed. + +template < + typename T, + typename = std::enable_if_t< + std::is_same::value || std::is_same::value>> +struct vec2 { + T x, y; + typedef T scalar_t; + vec2(T x, T y) : x(x), y(y) {} +}; + +template +inline vec2 operator+(const vec2& a, const vec2& b) { + return vec2(a.x + b.x, a.y + b.y); +} + +template +inline vec2 operator-(const vec2& a, const vec2& b) { + return vec2(a.x - b.x, a.y - b.y); +} + +template +inline vec2 operator*(const T a, const vec2& b) { + return vec2(a * b.x, a * b.y); +} + +template +inline vec2 operator/(const vec2& a, const T b) { + if (b == 0.0) { + AT_ERROR( + "denominator in vec2 division is 0"); // prevent divide by 0 errors. + } + return vec2(a.x / b, a.y / b); +} + +template +inline T dot(const vec2& a, const vec2& b) { + return a.x * b.x + a.y * b.y; +} + +template +inline T norm(const vec2& a, const vec2& b) { + const vec2 ba = b - a; + return sqrt(dot(ba, ba)); +} + +template +std::ostream& operator<<(std::ostream& os, const vec2& v) { + os << "vec2(" << v.x << ", " << v.y << ")"; + return os; +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/vec3.h b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/vec3.h new file mode 100644 index 0000000000000000000000000000000000000000..fc37bf5c0dc0f041e58bd801dbf4b7d36eb1979b --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/vec3.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +// A fixed-sized vector with basic arithmetic operators useful for +// representing 3D coordinates. +// TODO: switch to Eigen if more functionality is needed. + +template < + typename T, + typename = std::enable_if_t< + std::is_same::value || std::is_same::value>> +struct vec3 { + T x, y, z; + typedef T scalar_t; + vec3(T x, T y, T z) : x(x), y(y), z(z) {} +}; + +template +inline vec3 operator+(const vec3& a, const vec3& b) { + return vec3(a.x + b.x, a.y + b.y, a.z + b.z); +} + +template +inline vec3 operator-(const vec3& a, const vec3& b) { + return vec3(a.x - b.x, a.y - b.y, a.z - b.z); +} + +template +inline vec3 operator/(const vec3& a, const T b) { + if (b == 0.0) { + AT_ERROR( + "denominator in vec3 division is 0"); // prevent divide by 0 errors. + } + return vec3(a.x / b, a.y / b, a.z / b); +} + +template +inline vec3 operator*(const T a, const vec3& b) { + return vec3(a * b.x, a * b.y, a * b.z); +} + +template +inline vec3 operator*(const vec3& a, const vec3& b) { + return vec3(a.x * b.x, a.y * b.y, a.z * b.z); +} + +template +inline T dot(const vec3& a, const vec3& b) { + return a.x * b.x + a.y * b.y + a.z * b.z; +} + +template +inline vec3 cross(const vec3& a, const vec3& b) { + return vec3( + a.y * b.z - a.z * b.y, a.z * b.x - a.x * b.z, a.x * b.y - a.y * b.x); +} + +template +inline T norm(const vec3& a) { + return sqrt(dot(a, a)); +} + +template +std::ostream& operator<<(std::ostream& os, const vec3& v) { + os << "vec3(" << v.x << ", " << v.y << ", " << v.z << ")"; + return os; +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/warp_reduce.cuh b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/warp_reduce.cuh new file mode 100644 index 0000000000000000000000000000000000000000..3c903019debf5db594a6c71e1296ccd764991736 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/csrc/utils/warp_reduce.cuh @@ -0,0 +1,94 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include + +// Helper functions WarpReduceMin and WarpReduceMax used in .cu files +// Starting in Volta, instructions are no longer synchronous within a warp. +// We need to call __syncwarp() to sync the 32 threads in the warp +// instead of all the threads in the block. + +template +__device__ void +WarpReduceMin(scalar_t* min_dists, int64_t* min_idxs, const size_t tid) { + // s = 32 + if (min_dists[tid] > min_dists[tid + 32]) { + min_idxs[tid] = min_idxs[tid + 32]; + min_dists[tid] = min_dists[tid + 32]; + } + __syncwarp(); + // s = 16 + if (min_dists[tid] > min_dists[tid + 16]) { + min_idxs[tid] = min_idxs[tid + 16]; + min_dists[tid] = min_dists[tid + 16]; + } + __syncwarp(); + // s = 8 + if (min_dists[tid] > min_dists[tid + 8]) { + min_idxs[tid] = min_idxs[tid + 8]; + min_dists[tid] = min_dists[tid + 8]; + } + __syncwarp(); + // s = 4 + if (min_dists[tid] > min_dists[tid + 4]) { + min_idxs[tid] = min_idxs[tid + 4]; + min_dists[tid] = min_dists[tid + 4]; + } + __syncwarp(); + // s = 2 + if (min_dists[tid] > min_dists[tid + 2]) { + min_idxs[tid] = min_idxs[tid + 2]; + min_dists[tid] = min_dists[tid + 2]; + } + __syncwarp(); + // s = 1 + if (min_dists[tid] > min_dists[tid + 1]) { + min_idxs[tid] = min_idxs[tid + 1]; + min_dists[tid] = min_dists[tid + 1]; + } + __syncwarp(); +} + +template +__device__ void WarpReduceMax( + volatile scalar_t* dists, + volatile int64_t* dists_idx, + const size_t tid) { + if (dists[tid] < dists[tid + 32]) { + dists[tid] = dists[tid + 32]; + dists_idx[tid] = dists_idx[tid + 32]; + } + __syncwarp(); + if (dists[tid] < dists[tid + 16]) { + dists[tid] = dists[tid + 16]; + dists_idx[tid] = dists_idx[tid + 16]; + } + __syncwarp(); + if (dists[tid] < dists[tid + 8]) { + dists[tid] = dists[tid + 8]; + dists_idx[tid] = dists_idx[tid + 8]; + } + __syncwarp(); + if (dists[tid] < dists[tid + 4]) { + dists[tid] = dists[tid + 4]; + dists_idx[tid] = dists_idx[tid + 4]; + } + __syncwarp(); + if (dists[tid] < dists[tid + 2]) { + dists[tid] = dists[tid + 2]; + dists_idx[tid] = dists_idx[tid + 2]; + } + __syncwarp(); + if (dists[tid] < dists[tid + 1]) { + dists[tid] = dists[tid + 1]; + dists_idx[tid] = dists_idx[tid + 1]; + } + __syncwarp(); +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/__init__.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..47f8852be662f6233e9c2dac4b9cdb91d82a0b7c --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from .r2n2 import BlenderCamera, collate_batched_R2N2, R2N2, render_cubified_voxels +from .shapenet import ShapeNetCore +from .utils import collate_batched_meshes + + +__all__ = [k for k in globals().keys() if not k.startswith("_")] diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/r2n2/__init__.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/r2n2/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0b554d84b19b8dc58c49e438bc6a8950d27a9e42 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/r2n2/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from .r2n2 import R2N2 +from .utils import BlenderCamera, collate_batched_R2N2, render_cubified_voxels + + +__all__ = [k for k in globals().keys() if not k.startswith("_")] diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/r2n2/r2n2.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/r2n2/r2n2.py new file mode 100644 index 0000000000000000000000000000000000000000..c4e182f97890204be8d83d5d127e105da26c3499 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/r2n2/r2n2.py @@ -0,0 +1,427 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import json +import warnings +from os import path +from pathlib import Path +from typing import Dict, List, Optional + +import numpy as np +import torch +from PIL import Image +from pytorch3d.common.datatypes import Device +from pytorch3d.datasets.shapenet_base import ShapeNetBase +from pytorch3d.renderer import HardPhongShader +from tabulate import tabulate + +from .utils import ( + align_bbox, + BlenderCamera, + compute_extrinsic_matrix, + read_binvox_coords, + voxelize, +) + + +SYNSET_DICT_DIR = Path(__file__).resolve().parent +MAX_CAMERA_DISTANCE = 1.75 # Constant from R2N2. +VOXEL_SIZE = 128 +# Intrinsic matrix extracted from Blender. Taken from meshrcnn codebase: +# https://github.com/facebookresearch/meshrcnn/blob/main/shapenet/utils/coords.py +BLENDER_INTRINSIC = torch.tensor( + [ + [2.1875, 0.0, 0.0, 0.0], + [0.0, 2.1875, 0.0, 0.0], + [0.0, 0.0, -1.002002, -0.2002002], + [0.0, 0.0, -1.0, 0.0], + ] +) + + +class R2N2(ShapeNetBase): # pragma: no cover + """ + This class loads the R2N2 dataset from a given directory into a Dataset object. + The R2N2 dataset contains 13 categories that are a subset of the ShapeNetCore v.1 + dataset. The R2N2 dataset also contains its own 24 renderings of each object and + voxelized models. Most of the models have all 24 views in the same split, but there + are eight of them that divide their views between train and test splits. + + """ + + def __init__( + self, + split: str, + shapenet_dir: str, + r2n2_dir: str, + splits_file: str, + return_all_views: bool = True, + return_voxels: bool = False, + views_rel_path: str = "ShapeNetRendering", + voxels_rel_path: str = "ShapeNetVoxels", + load_textures: bool = True, + texture_resolution: int = 4, + ) -> None: + """ + Store each object's synset id and models id the given directories. + + Args: + split (str): One of (train, val, test). + shapenet_dir (str): Path to ShapeNet core v1. + r2n2_dir (str): Path to the R2N2 dataset. + splits_file (str): File containing the train/val/test splits. + return_all_views (bool): Indicator of whether or not to load all the views in + the split. If set to False, one of the views in the split will be randomly + selected and loaded. + return_voxels(bool): Indicator of whether or not to return voxels as a tensor + of shape (D, D, D) where D is the number of voxels along each dimension. + views_rel_path: path to rendered views within the r2n2_dir. If not specified, + the renderings are assumed to be at os.path.join(rn2n_dir, "ShapeNetRendering"). + voxels_rel_path: path to rendered views within the r2n2_dir. If not specified, + the renderings are assumed to be at os.path.join(rn2n_dir, "ShapeNetVoxels"). + load_textures: Boolean indicating whether textures should loaded for the model. + Textures will be of type TexturesAtlas i.e. a texture map per face. + texture_resolution: Int specifying the resolution of the texture map per face + created using the textures in the obj file. A + (texture_resolution, texture_resolution, 3) map is created per face. + + """ + super().__init__() + self.shapenet_dir = shapenet_dir + self.r2n2_dir = r2n2_dir + self.views_rel_path = views_rel_path + self.voxels_rel_path = voxels_rel_path + self.load_textures = load_textures + self.texture_resolution = texture_resolution + # Examine if split is valid. + if split not in ["train", "val", "test"]: + raise ValueError("split has to be one of (train, val, test).") + # Synset dictionary mapping synset offsets in R2N2 to corresponding labels. + with open( + path.join(SYNSET_DICT_DIR, "r2n2_synset_dict.json"), "r" + ) as read_dict: + self.synset_dict = json.load(read_dict) + # Inverse dictionary mapping synset labels to corresponding offsets. + self.synset_inv = {label: offset for offset, label in self.synset_dict.items()} + + # Store synset and model ids of objects mentioned in the splits_file. + with open(splits_file) as splits: + split_dict = json.load(splits)[split] + + self.return_images = True + # Check if the folder containing R2N2 renderings is included in r2n2_dir. + if not path.isdir(path.join(r2n2_dir, views_rel_path)): + self.return_images = False + msg = ( + "%s not found in %s. R2N2 renderings will " + "be skipped when returning models." + ) % (views_rel_path, r2n2_dir) + warnings.warn(msg) + + self.return_voxels = return_voxels + # Check if the folder containing voxel coordinates is included in r2n2_dir. + if not path.isdir(path.join(r2n2_dir, voxels_rel_path)): + self.return_voxels = False + msg = ( + "%s not found in %s. Voxel coordinates will " + "be skipped when returning models." + ) % (voxels_rel_path, r2n2_dir) + warnings.warn(msg) + + synset_set = set() + # Store lists of views of each model in a list. + self.views_per_model_list = [] + # Store tuples of synset label and total number of views in each category in a list. + synset_num_instances = [] + for synset in split_dict.keys(): + # Examine if the given synset is present in the ShapeNetCore dataset + # and is also part of the standard R2N2 dataset. + if not ( + path.isdir(path.join(shapenet_dir, synset)) + and synset in self.synset_dict + ): + msg = ( + "Synset category %s from the splits file is either not " + "present in %s or not part of the standard R2N2 dataset." + ) % (synset, shapenet_dir) + warnings.warn(msg) + continue + + synset_set.add(synset) + self.synset_start_idxs[synset] = len(self.synset_ids) + # Start counting total number of views in the current category. + synset_view_count = 0 + for model in split_dict[synset]: + # Examine if the given model is present in the ShapeNetCore path. + shapenet_path = path.join(shapenet_dir, synset, model) + if not path.isdir(shapenet_path): + msg = "Model %s from category %s is not present in %s." % ( + model, + synset, + shapenet_dir, + ) + warnings.warn(msg) + continue + self.synset_ids.append(synset) + self.model_ids.append(model) + + model_views = split_dict[synset][model] + # Randomly select a view index if return_all_views set to False. + if not return_all_views: + rand_idx = torch.randint(len(model_views), (1,)) + model_views = [model_views[rand_idx]] + self.views_per_model_list.append(model_views) + synset_view_count += len(model_views) + synset_num_instances.append((self.synset_dict[synset], synset_view_count)) + model_count = len(self.synset_ids) - self.synset_start_idxs[synset] + self.synset_num_models[synset] = model_count + headers = ["category", "#instances"] + synset_num_instances.append(("total", sum(n for _, n in synset_num_instances))) + print( + tabulate(synset_num_instances, headers, numalign="left", stralign="center") + ) + + # Examine if all the synsets in the standard R2N2 mapping are present. + # Update self.synset_inv so that it only includes the loaded categories. + synset_not_present = [ + self.synset_inv.pop(self.synset_dict[synset]) + for synset in self.synset_dict + if synset not in synset_set + ] + if len(synset_not_present) > 0: + msg = ( + "The following categories are included in R2N2's" + "official mapping but not found in the dataset location %s: %s" + ) % (shapenet_dir, ", ".join(synset_not_present)) + warnings.warn(msg) + + def __getitem__(self, model_idx, view_idxs: Optional[List[int]] = None) -> Dict: + """ + Read a model by the given index. + + Args: + model_idx: The idx of the model to be retrieved in the dataset. + view_idx: List of indices of the view to be returned. Each index needs to be + contained in the loaded split (always between 0 and 23, inclusive). If + an invalid index is supplied, view_idx will be ignored and all the loaded + views will be returned. + + Returns: + dictionary with following keys: + - verts: FloatTensor of shape (V, 3). + - faces: faces.verts_idx, LongTensor of shape (F, 3). + - synset_id (str): synset id. + - model_id (str): model id. + - label (str): synset label. + - images: FloatTensor of shape (V, H, W, C), where V is number of views + returned. Returns a batch of the renderings of the models from the R2N2 dataset. + - R: Rotation matrix of shape (V, 3, 3), where V is number of views returned. + - T: Translation matrix of shape (V, 3), where V is number of views returned. + - K: Intrinsic matrix of shape (V, 4, 4), where V is number of views returned. + - voxels: Voxels of shape (D, D, D), where D is the number of voxels along each + dimension. + """ + if isinstance(model_idx, tuple): + model_idx, view_idxs = model_idx + if view_idxs is not None: + if isinstance(view_idxs, int): + view_idxs = [view_idxs] + if not isinstance(view_idxs, list) and not torch.is_tensor(view_idxs): + raise TypeError( + "view_idxs is of type %s but it needs to be a list." + % type(view_idxs) + ) + + model_views = self.views_per_model_list[model_idx] + if view_idxs is not None and any( + idx not in self.views_per_model_list[model_idx] for idx in view_idxs + ): + msg = """At least one of the indices in view_idxs is not available. + Specified view of the model needs to be contained in the + loaded split. If return_all_views is set to False, only one + random view is loaded. Try accessing the specified view(s) + after loading the dataset with self.return_all_views set to True. + Now returning all view(s) in the loaded dataset.""" + warnings.warn(msg) + elif view_idxs is not None: + model_views = view_idxs + + model = self._get_item_ids(model_idx) + model_path = path.join( + self.shapenet_dir, model["synset_id"], model["model_id"], "model.obj" + ) + + verts, faces, textures = self._load_mesh(model_path) + model["verts"] = verts + model["faces"] = faces + model["textures"] = textures + model["label"] = self.synset_dict[model["synset_id"]] + + model["images"] = None + images, Rs, Ts, voxel_RTs = [], [], [], [] + # Retrieve R2N2's renderings if required. + if self.return_images: + rendering_path = path.join( + self.r2n2_dir, + self.views_rel_path, + model["synset_id"], + model["model_id"], + "rendering", + ) + # Read metadata file to obtain params for calibration matrices. + with open(path.join(rendering_path, "rendering_metadata.txt"), "r") as f: + metadata_lines = f.readlines() + for i in model_views: + # Read image. + image_path = path.join(rendering_path, "%02d.png" % i) + raw_img = Image.open(image_path) + image = torch.from_numpy(np.array(raw_img) / 255.0)[..., :3] + images.append(image.to(dtype=torch.float32)) + + # Get camera calibration. + azim, elev, yaw, dist_ratio, fov = [ + float(v) for v in metadata_lines[i].strip().split(" ") + ] + dist = dist_ratio * MAX_CAMERA_DISTANCE + # Extrinsic matrix before transformation to PyTorch3D world space. + RT = compute_extrinsic_matrix(azim, elev, dist) + R, T = self._compute_camera_calibration(RT) + Rs.append(R) + Ts.append(T) + voxel_RTs.append(RT) + + # Intrinsic matrix extracted from the Blender with slight modification to work with + # PyTorch3D world space. Taken from meshrcnn codebase: + # https://github.com/facebookresearch/meshrcnn/blob/main/shapenet/utils/coords.py + K = torch.tensor( + [ + [2.1875, 0.0, 0.0, 0.0], + [0.0, 2.1875, 0.0, 0.0], + [0.0, 0.0, -1.002002, -0.2002002], + [0.0, 0.0, 1.0, 0.0], + ] + ) + model["images"] = torch.stack(images) + model["R"] = torch.stack(Rs) + model["T"] = torch.stack(Ts) + model["K"] = K.expand(len(model_views), 4, 4) + + voxels_list = [] + + # Read voxels if required. + voxel_path = path.join( + self.r2n2_dir, + self.voxels_rel_path, + model["synset_id"], + model["model_id"], + "model.binvox", + ) + if self.return_voxels: + if not path.isfile(voxel_path): + msg = "Voxel file not found for model %s from category %s." + raise FileNotFoundError(msg % (model["model_id"], model["synset_id"])) + + with open(voxel_path, "rb") as f: + # Read voxel coordinates as a tensor of shape (N, 3). + voxel_coords = read_binvox_coords(f) + # Align voxels to the same coordinate system as mesh verts. + voxel_coords = align_bbox(voxel_coords, model["verts"]) + for RT in voxel_RTs: + # Compute projection matrix. + P = BLENDER_INTRINSIC.mm(RT) + # Convert voxel coordinates of shape (N, 3) to voxels of shape (D, D, D). + voxels = voxelize(voxel_coords, P, VOXEL_SIZE) + voxels_list.append(voxels) + model["voxels"] = torch.stack(voxels_list) + + return model + + def _compute_camera_calibration(self, RT): + """ + Helper function for calculating rotation and translation matrices from ShapeNet + to camera transformation and ShapeNet to PyTorch3D transformation. + + Args: + RT: Extrinsic matrix that performs ShapeNet world view to camera view + transformation. + + Returns: + R: Rotation matrix of shape (3, 3). + T: Translation matrix of shape (3). + """ + # Transform the mesh vertices from shapenet world to pytorch3d world. + shapenet_to_pytorch3d = torch.tensor( + [ + [-1.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, -1.0, 0.0], + [0.0, 0.0, 0.0, 1.0], + ], + dtype=torch.float32, + ) + RT = torch.transpose(RT, 0, 1).mm(shapenet_to_pytorch3d) # (4, 4) + # Extract rotation and translation matrices from RT. + R = RT[:3, :3] + T = RT[3, :3] + return R, T + + def render( + self, + model_ids: Optional[List[str]] = None, + categories: Optional[List[str]] = None, + sample_nums: Optional[List[int]] = None, + idxs: Optional[List[int]] = None, + view_idxs: Optional[List[int]] = None, + shader_type=HardPhongShader, + device: Device = "cpu", + **kwargs, + ) -> torch.Tensor: + """ + Render models with BlenderCamera by default to achieve the same orientations as the + R2N2 renderings. Also accepts other types of cameras and any of the args that the + render function in the ShapeNetBase class accepts. + + Args: + view_idxs: each model will be rendered with the orientation(s) of the specified + views. Only render by view_idxs if no camera or args for BlenderCamera is + supplied. + Accepts any of the args of the render function in ShapeNetBase: + model_ids: List[str] of model_ids of models intended to be rendered. + categories: List[str] of categories intended to be rendered. categories + and sample_nums must be specified at the same time. categories can be given + in the form of synset offsets or labels, or a combination of both. + sample_nums: List[int] of number of models to be randomly sampled from + each category. Could also contain one single integer, in which case it + will be broadcasted for every category. + idxs: List[int] of indices of models to be rendered in the dataset. + shader_type: Shader to use for rendering. Examples include HardPhongShader + (default), SoftPhongShader etc or any other type of valid Shader class. + device: Device (as str or torch.device) on which the tensors should be located. + **kwargs: Accepts any of the kwargs that the renderer supports and any of the + args that BlenderCamera supports. + + Returns: + Batch of rendered images of shape (N, H, W, 3). + """ + idxs = self._handle_render_inputs(model_ids, categories, sample_nums, idxs) + r = torch.cat([self[idxs[i], view_idxs]["R"] for i in range(len(idxs))]) + t = torch.cat([self[idxs[i], view_idxs]["T"] for i in range(len(idxs))]) + k = torch.cat([self[idxs[i], view_idxs]["K"] for i in range(len(idxs))]) + # Initialize default camera using R, T, K from kwargs or R, T, K of the specified views. + blend_cameras = BlenderCamera( + R=kwargs.get("R", r), + T=kwargs.get("T", t), + K=kwargs.get("K", k), + device=device, + ) + cameras = kwargs.get("cameras", blend_cameras).to(device) + kwargs.pop("cameras", None) + # pass down all the same inputs + return super().render( + idxs=idxs, shader_type=shader_type, device=device, cameras=cameras, **kwargs + ) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/r2n2/r2n2_synset_dict.json b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/r2n2/r2n2_synset_dict.json new file mode 100644 index 0000000000000000000000000000000000000000..b8cbae58173e58ea0607e95161e65944979aff23 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/r2n2/r2n2_synset_dict.json @@ -0,0 +1,15 @@ +{ + "04256520": "sofa", + "02933112": "cabinet", + "02828884": "bench", + "03001627": "chair", + "03211117": "display", + "04090263": "rifle", + "03691459": "loudspeaker", + "03636649": "lamp", + "04401088": "telephone", + "02691156": "airplane", + "04379243": "table", + "02958343": "car", + "04530566": "watercraft" +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/r2n2/utils.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/r2n2/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0bb52ae15e4c687396e12ce1738fff921efbec65 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/r2n2/utils.py @@ -0,0 +1,504 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import math +from typing import Dict, List + +import numpy as np +import torch +from pytorch3d.common.datatypes import Device +from pytorch3d.datasets.utils import collate_batched_meshes +from pytorch3d.ops import cubify +from pytorch3d.renderer import ( + HardPhongShader, + MeshRasterizer, + MeshRenderer, + PointLights, + RasterizationSettings, + TexturesVertex, +) +from pytorch3d.renderer.cameras import CamerasBase +from pytorch3d.transforms import Transform3d + + +# Empirical min and max over the dataset from meshrcnn. +# https://github.com/facebookresearch/meshrcnn/blob/main/shapenet/utils/coords.py#L9 +SHAPENET_MIN_ZMIN = 0.67 +SHAPENET_MAX_ZMAX = 0.92 +# Threshold for cubify from meshrcnn: +# https://github.com/facebookresearch/meshrcnn/blob/main/configs/shapenet/voxmesh_R50.yaml#L11 +CUBIFY_THRESH = 0.2 + +# Default values of rotation, translation and intrinsic matrices for BlenderCamera. +r = np.expand_dims(np.eye(3), axis=0) # (1, 3, 3) +t = np.expand_dims(np.zeros(3), axis=0) # (1, 3) +k = np.expand_dims(np.eye(4), axis=0) # (1, 4, 4) + + +def collate_batched_R2N2(batch: List[Dict]): # pragma: no cover + """ + Take a list of objects in the form of dictionaries and merge them + into a single dictionary. This function can be used with a Dataset + object to create a torch.utils.data.Dataloader which directly + returns Meshes objects. + TODO: Add support for textures. + + Args: + batch: List of dictionaries containing information about objects + in the dataset. + + Returns: + collated_dict: Dictionary of collated lists. If batch contains both + verts and faces, a collated mesh batch is also returned. + """ + collated_dict = collate_batched_meshes(batch) + + # If collate_batched_meshes receives R2N2 items with images and that + # all models have the same number of views V, stack the batches of + # views of each model into a new batch of shape (N, V, H, W, 3). + # Otherwise leave it as a list. + if "images" in collated_dict: + try: + collated_dict["images"] = torch.stack(collated_dict["images"]) + except RuntimeError: + print( + "Models don't have the same number of views. Now returning " + "lists of images instead of batches." + ) + + # If collate_batched_meshes receives R2N2 items with camera calibration + # matrices and that all models have the same number of views V, stack each + # type of matrices into a new batch of shape (N, V, ...). + # Otherwise leave them as lists. + if all(x in collated_dict for x in ["R", "T", "K"]): + try: + collated_dict["R"] = torch.stack(collated_dict["R"]) # (N, V, 3, 3) + collated_dict["T"] = torch.stack(collated_dict["T"]) # (N, V, 3) + collated_dict["K"] = torch.stack(collated_dict["K"]) # (N, V, 4, 4) + except RuntimeError: + print( + "Models don't have the same number of views. Now returning " + "lists of calibration matrices instead of a batched tensor." + ) + + # If collate_batched_meshes receives voxels and all models have the same + # number of views V, stack the batches of voxels into a new batch of shape + # (N, V, S, S, S), where S is the voxel size. + if "voxels" in collated_dict: + try: + collated_dict["voxels"] = torch.stack(collated_dict["voxels"]) + except RuntimeError: + print( + "Models don't have the same number of views. Now returning " + "lists of voxels instead of a batched tensor." + ) + return collated_dict + + +def compute_extrinsic_matrix( + azimuth: float, elevation: float, distance: float +): # pragma: no cover + """ + Copied from meshrcnn codebase: + https://github.com/facebookresearch/meshrcnn/blob/main/shapenet/utils/coords.py#L96 + + Compute 4x4 extrinsic matrix that converts from homogeneous world coordinates + to homogeneous camera coordinates. We assume that the camera is looking at the + origin. + Used in R2N2 Dataset when computing calibration matrices. + + Args: + azimuth: Rotation about the z-axis, in degrees. + elevation: Rotation above the xy-plane, in degrees. + distance: Distance from the origin. + + Returns: + FloatTensor of shape (4, 4). + """ + azimuth, elevation, distance = float(azimuth), float(elevation), float(distance) + + az_rad = -math.pi * azimuth / 180.0 + el_rad = -math.pi * elevation / 180.0 + sa = math.sin(az_rad) + ca = math.cos(az_rad) + se = math.sin(el_rad) + ce = math.cos(el_rad) + R_world2obj = torch.tensor( + [[ca * ce, sa * ce, -se], [-sa, ca, 0], [ca * se, sa * se, ce]] + ) + R_obj2cam = torch.tensor([[0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [1.0, 0.0, 0.0]]) + R_world2cam = R_obj2cam.mm(R_world2obj) + cam_location = torch.tensor([[distance, 0, 0]]).t() + T_world2cam = -(R_obj2cam.mm(cam_location)) + RT = torch.cat([R_world2cam, T_world2cam], dim=1) + RT = torch.cat([RT, torch.tensor([[0.0, 0, 0, 1]])]) + + # Georgia: For some reason I cannot fathom, when Blender loads a .obj file it + # rotates the model 90 degrees about the x axis. To compensate for this quirk we + # roll that rotation into the extrinsic matrix here + rot = torch.tensor([[1, 0, 0, 0], [0, 0, -1, 0], [0, 1, 0, 0], [0, 0, 0, 1]]) + RT = RT.mm(rot.to(RT)) + + return RT + + +def read_binvox_coords( + f, + integer_division: bool = True, + dtype: torch.dtype = torch.float32, +): # pragma: no cover + """ + Copied from meshrcnn codebase: + https://github.com/facebookresearch/meshrcnn/blob/main/shapenet/utils/binvox_torch.py#L5 + + Read a binvox file and return the indices of all nonzero voxels. + + This matches the behavior of binvox_rw.read_as_coord_array + (https://github.com/dimatura/binvox-rw-py/blob/public/binvox_rw.py#L153) + but this implementation uses torch rather than numpy, and is more efficient + due to improved vectorization. + + Georgia: I think that binvox_rw.read_as_coord_array actually has a bug; when converting + linear indices into three-dimensional indices, they use floating-point + division instead of integer division. We can reproduce their incorrect + implementation by passing integer_division=False. + + Args: + f (str): A file pointer to the binvox file to read + integer_division (bool): If False, then match the buggy implementation from binvox_rw + dtype: Datatype of the output tensor. Use float64 to match binvox_rw + + Returns: + coords (tensor): A tensor of shape (N, 3) where N is the number of nonzero voxels, + and coords[i] = (x, y, z) gives the index of the ith nonzero voxel. If the + voxel grid has shape (V, V, V) then we have 0 <= x, y, z < V. + """ + size, translation, scale = _read_binvox_header(f) + storage = torch.ByteStorage.from_buffer(f.read()) + data = torch.tensor([], dtype=torch.uint8) + # pyre-fixme[28]: Unexpected keyword argument `source`. + data.set_(source=storage) + vals, counts = data[::2], data[1::2] + idxs = _compute_idxs(vals, counts) + if not integer_division: + idxs = idxs.to(dtype) + x_idxs = idxs // (size * size) + zy_idxs = idxs % (size * size) + z_idxs = zy_idxs // size + y_idxs = zy_idxs % size + coords = torch.stack([x_idxs, y_idxs, z_idxs], dim=1) + return coords.to(dtype) + + +def _compute_idxs(vals, counts): # pragma: no cover + """ + Copied from meshrcnn codebase: + https://github.com/facebookresearch/meshrcnn/blob/main/shapenet/utils/binvox_torch.py#L58 + + Fast vectorized version of index computation. + + Args: + vals: tensor of binary values indicating voxel presence in a dense format. + counts: tensor of number of occurrence of each value in vals. + + Returns: + idxs: A tensor of shape (N), where N is the number of nonzero voxels. + """ + # Consider an example where: + # vals = [0, 1, 0, 1, 1] + # counts = [2, 3, 3, 2, 1] + # + # These values of counts and vals mean that the dense binary grid is: + # [0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1] + # + # So the nonzero indices we want to return are: + # [2, 3, 4, 8, 9, 10] + + # After the cumsum we will have: + # end_idxs = [2, 5, 8, 10, 11] + end_idxs = counts.cumsum(dim=0) + + # After masking and computing start_idx we have: + # end_idxs = [5, 10, 11] + # counts = [3, 2, 1] + # start_idxs = [2, 8, 10] + mask = vals == 1 + end_idxs = end_idxs[mask] + counts = counts[mask].to(end_idxs) + start_idxs = end_idxs - counts + + # We initialize delta as: + # [2, 1, 1, 1, 1, 1] + delta = torch.ones(counts.sum().item(), dtype=torch.int64) + delta[0] = start_idxs[0] + + # We compute pos = [3, 5], val = [3, 0]; then delta is + # [2, 1, 1, 4, 1, 1] + pos = counts.cumsum(dim=0)[:-1] + val = start_idxs[1:] - end_idxs[:-1] + delta[pos] += val + + # A final cumsum gives the idx we want: [2, 3, 4, 8, 9, 10] + idxs = delta.cumsum(dim=0) + return idxs + + +def _read_binvox_header(f): # pragma: no cover + """ + Copied from meshrcnn codebase: + https://github.com/facebookresearch/meshrcnn/blob/main/shapenet/utils/binvox_torch.py#L99 + + Read binvox header and extract information regarding voxel sizes and translations + to original voxel coordinates. + + Args: + f (str): A file pointer to the binvox file to read. + + Returns: + size (int): size of voxel. + translation (tuple(float)): translation to original voxel coordinates. + scale (float): scale to original voxel coordinates. + """ + # First line of the header should be "#binvox 1" + line = f.readline().strip() + if line != b"#binvox 1": + raise ValueError("Invalid header (line 1)") + + # Second line of the header should be "dim [int] [int] [int]" + # and all three int should be the same + line = f.readline().strip() + if not line.startswith(b"dim "): + raise ValueError("Invalid header (line 2)") + dims = line.split(b" ") + try: + dims = [int(d) for d in dims[1:]] + except ValueError: + raise ValueError("Invalid header (line 2)") from None + if len(dims) != 3 or dims[0] != dims[1] or dims[0] != dims[2]: + raise ValueError("Invalid header (line 2)") + size = dims[0] + + # Third line of the header should be "translate [float] [float] [float]" + line = f.readline().strip() + if not line.startswith(b"translate "): + raise ValueError("Invalid header (line 3)") + translation = line.split(b" ") + if len(translation) != 4: + raise ValueError("Invalid header (line 3)") + try: + translation = tuple(float(t) for t in translation[1:]) + except ValueError: + raise ValueError("Invalid header (line 3)") from None + + # Fourth line of the header should be "scale [float]" + line = f.readline().strip() + if not line.startswith(b"scale "): + raise ValueError("Invalid header (line 4)") + line = line.split(b" ") + if not len(line) == 2: + raise ValueError("Invalid header (line 4)") + scale = float(line[1]) + + # Fifth line of the header should be "data" + line = f.readline().strip() + if not line == b"data": + raise ValueError("Invalid header (line 5)") + + return size, translation, scale + + +def align_bbox(src, tgt): # pragma: no cover + """ + Copied from meshrcnn codebase: + https://github.com/facebookresearch/meshrcnn/blob/main/tools/preprocess_shapenet.py#L263 + + Return a copy of src points in the coordinate system of tgt by applying a + scale and shift along each coordinate axis to make the min / max values align. + + Args: + src, tgt: Torch Tensor of shape (N, 3) + + Returns: + out: Torch Tensor of shape (N, 3) + """ + if src.ndim != 2 or tgt.ndim != 2: + raise ValueError("Both src and tgt need to have dimensions of 2.") + if src.shape[-1] != 3 or tgt.shape[-1] != 3: + raise ValueError( + "Both src and tgt need to have sizes of 3 along the second dimension." + ) + src_min = src.min(dim=0)[0] + src_max = src.max(dim=0)[0] + tgt_min = tgt.min(dim=0)[0] + tgt_max = tgt.max(dim=0)[0] + scale = (tgt_max - tgt_min) / (src_max - src_min) + shift = tgt_min - scale * src_min + out = scale * src + shift + return out + + +def voxelize(voxel_coords, P, V): # pragma: no cover + """ + Copied from meshrcnn codebase: + https://github.com/facebookresearch/meshrcnn/blob/main/tools/preprocess_shapenet.py#L284 + but changing flip y to flip x. + + Creating voxels of shape (D, D, D) from voxel_coords and projection matrix. + + Args: + voxel_coords: FloatTensor of shape (V, 3) giving voxel's coordinates aligned to + the vertices. + P: FloatTensor of shape (4, 4) giving the projection matrix. + V: Voxel size of the output. + + Returns: + voxels: Tensor of shape (D, D, D) giving the voxelized result. + """ + device = voxel_coords.device + voxel_coords = project_verts(voxel_coords, P) + + # Using the actual zmin and zmax of the model is bad because we need them + # to perform the inverse transform, which transform voxels back into world + # space for refinement or evaluation. Instead we use an empirical min and + # max over the dataset; that way it is consistent for all images. + zmin = SHAPENET_MIN_ZMIN + zmax = SHAPENET_MAX_ZMAX + + # Once we know zmin and zmax, we need to adjust the z coordinates so the + # range [zmin, zmax] instead runs from [-1, 1] + m = 2.0 / (zmax - zmin) + b = -2.0 * zmin / (zmax - zmin) - 1 + voxel_coords[:, 2].mul_(m).add_(b) + voxel_coords[:, 0].mul_(-1) # Flip x + + # Now voxels are in [-1, 1]^3; map to [0, V-1)^3 + voxel_coords = 0.5 * (V - 1) * (voxel_coords + 1.0) + voxel_coords = voxel_coords.round().to(torch.int64) + valid = (0 <= voxel_coords) * (voxel_coords < V) + valid = valid[:, 0] * valid[:, 1] * valid[:, 2] + x, y, z = voxel_coords.unbind(dim=1) + x, y, z = x[valid], y[valid], z[valid] + voxels = torch.zeros(V, V, V, dtype=torch.uint8, device=device) + voxels[z, y, x] = 1 + + return voxels + + +def project_verts(verts, P, eps: float = 1e-1): # pragma: no cover + """ + Copied from meshrcnn codebase: + https://github.com/facebookresearch/meshrcnn/blob/main/shapenet/utils/coords.py#L159 + + Project vertices using a 4x4 transformation matrix. + + Args: + verts: FloatTensor of shape (N, V, 3) giving a batch of vertex positions or of + shape (V, 3) giving a single set of vertex positions. + P: FloatTensor of shape (N, 4, 4) giving projection matrices or of shape (4, 4) + giving a single projection matrix. + + Returns: + verts_out: FloatTensor of shape (N, V, 3) giving vertex positions (x, y, z) + where verts_out[i] is the result of transforming verts[i] by P[i]. + """ + # Handle unbatched inputs + singleton = False + if verts.dim() == 2: + assert P.dim() == 2 + singleton = True + verts, P = verts[None], P[None] + + N, V = verts.shape[0], verts.shape[1] + dtype, device = verts.dtype, verts.device + + # Add an extra row of ones to the world-space coordinates of verts before + # multiplying by the projection matrix. We could avoid this allocation by + # instead multiplying by a 4x3 submatrix of the projection matrix, then + # adding the remaining 4x1 vector. Not sure whether there will be much + # performance difference between the two. + ones = torch.ones(N, V, 1, dtype=dtype, device=device) + verts_hom = torch.cat([verts, ones], dim=2) + verts_cam_hom = torch.bmm(verts_hom, P.transpose(1, 2)) + + # Avoid division by zero by clamping the absolute value + w = verts_cam_hom[:, :, 3:] + w_sign = w.sign() + w_sign[w == 0] = 1 + w = w_sign * w.abs().clamp(min=eps) + + verts_proj = verts_cam_hom[:, :, :3] / w + + if singleton: + return verts_proj[0] + return verts_proj + + +class BlenderCamera(CamerasBase): # pragma: no cover + """ + Camera for rendering objects with calibration matrices from the R2N2 dataset + (which uses Blender for rendering the views for each model). + """ + + def __init__(self, R=r, T=t, K=k, device: Device = "cpu") -> None: + """ + Args: + R: Rotation matrix of shape (N, 3, 3). + T: Translation matrix of shape (N, 3). + K: Intrinsic matrix of shape (N, 4, 4). + device: Device (as str or torch.device). + """ + # The initializer formats all inputs to torch tensors and broadcasts + # all the inputs to have the same batch dimension where necessary. + super().__init__(device=device, R=R, T=T, K=K) + + def get_projection_transform(self, **kwargs) -> Transform3d: + transform = Transform3d(device=self.device) + transform._matrix = self.K.transpose(1, 2).contiguous() + return transform + + def is_perspective(self): + return False + + def in_ndc(self): + return True + + +def render_cubified_voxels( + voxels: torch.Tensor, shader_type=HardPhongShader, device: Device = "cpu", **kwargs +): # pragma: no cover + """ + Use the Cubify operator to convert inputs voxels to a mesh and then render that mesh. + + Args: + voxels: FloatTensor of shape (N, D, D, D) where N is the batch size and + D is the number of voxels along each dimension. + shader_type: shader_type: shader_type: Shader to use for rendering. Examples + include HardPhongShader (default), SoftPhongShader etc or any other type + of valid Shader class. + device: Device (as str or torch.device) on which the tensors should be located. + **kwargs: Accepts any of the kwargs that the renderer supports. + Returns: + Batch of rendered images of shape (N, H, W, 3). + """ + cubified_voxels = cubify(voxels, CUBIFY_THRESH).to(device) + cubified_voxels.textures = TexturesVertex( + verts_features=torch.ones_like(cubified_voxels.verts_padded(), device=device) + ) + cameras = BlenderCamera(device=device) + renderer = MeshRenderer( + rasterizer=MeshRasterizer( + cameras=cameras, + raster_settings=kwargs.get("raster_settings", RasterizationSettings()), + ), + shader=shader_type( + device=device, + cameras=cameras, + lights=kwargs.get("lights", PointLights()).to(device), + ), + ) + return renderer(cubified_voxels) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/shapenet/__init__.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/shapenet/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1c2f532ea35ebc74f272bfafef57e6b009eba299 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/shapenet/__init__.py @@ -0,0 +1,12 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from .shapenet_core import ShapeNetCore + + +__all__ = [k for k in globals().keys() if not k.startswith("_")] diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/shapenet/shapenet_core.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/shapenet/shapenet_core.py new file mode 100644 index 0000000000000000000000000000000000000000..70524b15dc6f460762334ef0c9b7d1aba7af4e01 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/shapenet/shapenet_core.py @@ -0,0 +1,160 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import json +import os +import warnings +from os import path +from pathlib import Path +from typing import Dict + +from pytorch3d.datasets.shapenet_base import ShapeNetBase + + +SYNSET_DICT_DIR = Path(__file__).resolve().parent + + +class ShapeNetCore(ShapeNetBase): # pragma: no cover + """ + This class loads ShapeNetCore from a given directory into a Dataset object. + ShapeNetCore is a subset of the ShapeNet dataset and can be downloaded from + https://www.shapenet.org/. + """ + + def __init__( + self, + data_dir, + synsets=None, + version: int = 1, + load_textures: bool = True, + texture_resolution: int = 4, + ) -> None: + """ + Store each object's synset id and models id from data_dir. + + Args: + data_dir: Path to ShapeNetCore data. + synsets: List of synset categories to load from ShapeNetCore in the form of + synset offsets or labels. A combination of both is also accepted. + When no category is specified, all categories in data_dir are loaded. + version: (int) version of ShapeNetCore data in data_dir, 1 or 2. + Default is set to be 1. Version 1 has 57 categories and version 2 has 55 + categories. + Note: version 1 has two categories 02858304(boat) and 02992529(cellphone) + that are hyponyms of categories 04530566(watercraft) and 04401088(telephone) + respectively. You can combine the categories manually if needed. + Version 2 doesn't have 02858304(boat) or 02834778(bicycle) compared to + version 1. + load_textures: Boolean indicating whether textures should loaded for the model. + Textures will be of type TexturesAtlas i.e. a texture map per face. + texture_resolution: Int specifying the resolution of the texture map per face + created using the textures in the obj file. A + (texture_resolution, texture_resolution, 3) map is created per face. + """ + super().__init__() + self.shapenet_dir = data_dir + self.load_textures = load_textures + self.texture_resolution = texture_resolution + + if version not in [1, 2]: + raise ValueError("Version number must be either 1 or 2.") + self.model_dir = "model.obj" if version == 1 else "models/model_normalized.obj" + + # Synset dictionary mapping synset offsets to corresponding labels. + dict_file = "shapenet_synset_dict_v%d.json" % version + with open(path.join(SYNSET_DICT_DIR, dict_file), "r") as read_dict: + self.synset_dict = json.load(read_dict) + # Inverse dictionary mapping synset labels to corresponding offsets. + self.synset_inv = {label: offset for offset, label in self.synset_dict.items()} + + # If categories are specified, check if each category is in the form of either + # synset offset or synset label, and if the category exists in the given directory. + if synsets is not None: + # Set of categories to load in the form of synset offsets. + synset_set = set() + for synset in synsets: + if (synset in self.synset_dict.keys()) and ( + path.isdir(path.join(data_dir, synset)) + ): + synset_set.add(synset) + elif (synset in self.synset_inv.keys()) and ( + (path.isdir(path.join(data_dir, self.synset_inv[synset]))) + ): + synset_set.add(self.synset_inv[synset]) + else: + msg = ( + "Synset category %s either not part of ShapeNetCore dataset " + "or cannot be found in %s." + ) % (synset, data_dir) + warnings.warn(msg) + # If no category is given, load every category in the given directory. + # Ignore synset folders not included in the official mapping. + else: + synset_set = { + synset + for synset in os.listdir(data_dir) + if path.isdir(path.join(data_dir, synset)) + and synset in self.synset_dict + } + + # Check if there are any categories in the official mapping that are not loaded. + # Update self.synset_inv so that it only includes the loaded categories. + synset_not_present = set(self.synset_dict.keys()).difference(synset_set) + [self.synset_inv.pop(self.synset_dict[synset]) for synset in synset_not_present] + + if len(synset_not_present) > 0: + msg = ( + "The following categories are included in ShapeNetCore ver.%d's " + "official mapping but not found in the dataset location %s: %s" + "" + ) % (version, data_dir, ", ".join(synset_not_present)) + warnings.warn(msg) + + # Extract model_id of each object from directory names. + # Each grandchildren directory of data_dir contains an object, and the name + # of the directory is the object's model_id. + for synset in synset_set: + self.synset_start_idxs[synset] = len(self.synset_ids) + for model in os.listdir(path.join(data_dir, synset)): + if not path.exists(path.join(data_dir, synset, model, self.model_dir)): + msg = ( + "Object file not found in the model directory %s " + "under synset directory %s." + ) % (model, synset) + warnings.warn(msg) + continue + self.synset_ids.append(synset) + self.model_ids.append(model) + model_count = len(self.synset_ids) - self.synset_start_idxs[synset] + self.synset_num_models[synset] = model_count + + def __getitem__(self, idx: int) -> Dict: + """ + Read a model by the given index. + + Args: + idx: The idx of the model to be retrieved in the dataset. + + Returns: + dictionary with following keys: + - verts: FloatTensor of shape (V, 3). + - faces: LongTensor of shape (F, 3) which indexes into the verts tensor. + - synset_id (str): synset id + - model_id (str): model id + - label (str): synset label. + """ + model = self._get_item_ids(idx) + model_path = path.join( + self.shapenet_dir, model["synset_id"], model["model_id"], self.model_dir + ) + verts, faces, textures = self._load_mesh(model_path) + model["verts"] = verts + model["faces"] = faces + model["textures"] = textures + model["label"] = self.synset_dict[model["synset_id"]] + return model diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/shapenet/shapenet_synset_dict_v1.json b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/shapenet/shapenet_synset_dict_v1.json new file mode 100644 index 0000000000000000000000000000000000000000..b2fc62ae62107a81e078ec02432fb554ae8f1b41 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/shapenet/shapenet_synset_dict_v1.json @@ -0,0 +1,59 @@ +{ + "04379243": "table", + "02958343": "car", + "03001627": "chair", + "02691156": "airplane", + "04256520": "sofa", + "04090263": "rifle", + "03636649": "lamp", + "04530566": "watercraft", + "02828884": "bench", + "03691459": "loudspeaker", + "02933112": "cabinet", + "03211117": "display", + "04401088": "telephone", + "02924116": "bus", + "02808440": "bathtub", + "03467517": "guitar", + "03325088": "faucet", + "03046257": "clock", + "03991062": "flowerpot", + "03593526": "jar", + "02876657": "bottle", + "02871439": "bookshelf", + "03642806": "laptop", + "03624134": "knife", + "04468005": "train", + "02747177": "trash bin", + "03790512": "motorbike", + "03948459": "pistol", + "03337140": "file cabinet", + "02818832": "bed", + "03928116": "piano", + "04330267": "stove", + "03797390": "mug", + "02880940": "bowl", + "04554684": "washer", + "04004475": "printer", + "03513137": "helmet", + "03761084": "microwaves", + "04225987": "skateboard", + "04460130": "tower", + "02942699": "camera", + "02801938": "basket", + "02946921": "can", + "03938244": "pillow", + "03710193": "mailbox", + "03207941": "dishwasher", + "04099429": "rocket", + "02773838": "bag", + "02843684": "birdhouse", + "03261776": "earphone", + "03759954": "microphone", + "04074963": "remote", + "03085013": "keyboard", + "02834778": "bicycle", + "02954340": "cap", + "02858304": "boat", + "02992529": "mobile phone" +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/shapenet/shapenet_synset_dict_v2.json b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/shapenet/shapenet_synset_dict_v2.json new file mode 100644 index 0000000000000000000000000000000000000000..f0107c93c3535e2454070be1dcb622ac66899c90 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/shapenet/shapenet_synset_dict_v2.json @@ -0,0 +1,57 @@ +{ + "02691156": "airplane", + "02747177": "trash bin", + "02773838": "bag", + "02801938": "basket", + "02808440": "bathtub", + "02818832": "bed", + "02828884": "bench", + "02843684": "birdhouse", + "02871439": "bookshelf", + "02876657": "bottle", + "02880940": "bowl", + "02924116": "bus", + "02933112": "cabinet", + "02942699": "camera", + "02946921": "can", + "02954340": "cap", + "02958343": "car", + "02992529": "cellphone", + "03001627": "chair", + "03046257": "clock", + "03085013": "keyboard", + "03207941": "dishwasher", + "03211117": "display", + "03261776": "earphone", + "03325088": "faucet", + "03337140": "file cabinet", + "03467517": "guitar", + "03513137": "helmet", + "03593526": "jar", + "03624134": "knife", + "03636649": "lamp", + "03642806": "laptop", + "03691459": "loudspeaker", + "03710193": "mailbox", + "03759954": "microphone", + "03761084": "microwaves", + "03790512": "motorbike", + "03797390": "mug", + "03928116": "piano", + "03938244": "pillow", + "03948459": "pistol", + "03991062": "flowerpot", + "04004475": "printer", + "04074963": "remote", + "04090263": "rifle", + "04099429": "rocket", + "04225987": "skateboard", + "04256520": "sofa", + "04330267": "stove", + "04379243": "table", + "04401088": "telephone", + "04460130": "tower", + "04468005": "train", + "04530566": "watercraft", + "04554684": "washer" +} diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/shapenet_base.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/shapenet_base.py new file mode 100644 index 0000000000000000000000000000000000000000..d90bea67956019adbb0ea29dd7524f19060f88f8 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/shapenet_base.py @@ -0,0 +1,291 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import warnings +from typing import Dict, List, Optional, Tuple + +import torch +from pytorch3d.common.datatypes import Device +from pytorch3d.io import load_obj +from pytorch3d.renderer import ( + FoVPerspectiveCameras, + HardPhongShader, + MeshRasterizer, + MeshRenderer, + PointLights, + RasterizationSettings, + TexturesVertex, +) + +from .utils import collate_batched_meshes + + +class ShapeNetBase(torch.utils.data.Dataset): # pragma: no cover + """ + 'ShapeNetBase' implements a base Dataset for ShapeNet and R2N2 with helper methods. + It is not intended to be used on its own as a Dataset for a Dataloader. Both __init__ + and __getitem__ need to be implemented. + """ + + def __init__(self) -> None: + """ + Set up lists of synset_ids and model_ids. + """ + self.synset_ids = [] + self.model_ids = [] + self.synset_inv = {} + self.synset_start_idxs = {} + self.synset_num_models = {} + self.shapenet_dir = "" + self.model_dir = "model.obj" + self.load_textures = True + self.texture_resolution = 4 + + def __len__(self) -> int: + """ + Return number of total models in the loaded dataset. + """ + return len(self.model_ids) + + def __getitem__(self, idx) -> Dict: + """ + Read a model by the given index. Need to be implemented for every child class + of ShapeNetBase. + + Args: + idx: The idx of the model to be retrieved in the dataset. + + Returns: + dictionary containing information about the model. + """ + raise NotImplementedError( + "__getitem__ should be implemented in the child class of ShapeNetBase" + ) + + def _get_item_ids(self, idx) -> Dict: + """ + Read a model by the given index. + + Args: + idx: The idx of the model to be retrieved in the dataset. + + Returns: + dictionary with following keys: + - synset_id (str): synset id + - model_id (str): model id + """ + model = {} + model["synset_id"] = self.synset_ids[idx] + model["model_id"] = self.model_ids[idx] + return model + + def _load_mesh(self, model_path) -> Tuple: + verts, faces, aux = load_obj( + model_path, + create_texture_atlas=self.load_textures, + load_textures=self.load_textures, + texture_atlas_size=self.texture_resolution, + ) + if self.load_textures: + textures = aux.texture_atlas + # Some meshes don't have textures. In this case + # create a white texture map + if textures is None: + textures = verts.new_ones( + faces.verts_idx.shape[0], + self.texture_resolution, + self.texture_resolution, + 3, + ) + else: + textures = None + + return verts, faces.verts_idx, textures + + def render( + self, + model_ids: Optional[List[str]] = None, + categories: Optional[List[str]] = None, + sample_nums: Optional[List[int]] = None, + idxs: Optional[List[int]] = None, + shader_type=HardPhongShader, + device: Device = "cpu", + **kwargs, + ) -> torch.Tensor: + """ + If a list of model_ids are supplied, render all the objects by the given model_ids. + If no model_ids are supplied, but categories and sample_nums are specified, randomly + select a number of objects (number specified in sample_nums) in the given categories + and render these objects. If instead a list of idxs is specified, check if the idxs + are all valid and render models by the given idxs. Otherwise, randomly select a number + (first number in sample_nums, default is set to be 1) of models from the loaded dataset + and render these models. + + Args: + model_ids: List[str] of model_ids of models intended to be rendered. + categories: List[str] of categories intended to be rendered. categories + and sample_nums must be specified at the same time. categories can be given + in the form of synset offsets or labels, or a combination of both. + sample_nums: List[int] of number of models to be randomly sampled from + each category. Could also contain one single integer, in which case it + will be broadcasted for every category. + idxs: List[int] of indices of models to be rendered in the dataset. + shader_type: Select shading. Valid options include HardPhongShader (default), + SoftPhongShader, HardGouraudShader, SoftGouraudShader, HardFlatShader, + SoftSilhouetteShader. + device: Device (as str or torch.device) on which the tensors should be located. + **kwargs: Accepts any of the kwargs that the renderer supports. + + Returns: + Batch of rendered images of shape (N, H, W, 3). + """ + idxs = self._handle_render_inputs(model_ids, categories, sample_nums, idxs) + # Use the getitem method which loads mesh + texture + models = [self[idx] for idx in idxs] + meshes = collate_batched_meshes(models)["mesh"] + if meshes.textures is None: + meshes.textures = TexturesVertex( + verts_features=torch.ones_like(meshes.verts_padded(), device=device) + ) + + meshes = meshes.to(device) + cameras = kwargs.get("cameras", FoVPerspectiveCameras()).to(device) + if len(cameras) != 1 and len(cameras) % len(meshes) != 0: + raise ValueError("Mismatch between batch dims of cameras and meshes.") + if len(cameras) > 1: + # When rendering R2N2 models, if more than one views are provided, broadcast + # the meshes so that each mesh can be rendered for each of the views. + meshes = meshes.extend(len(cameras) // len(meshes)) + renderer = MeshRenderer( + rasterizer=MeshRasterizer( + cameras=cameras, + raster_settings=kwargs.get("raster_settings", RasterizationSettings()), + ), + shader=shader_type( + device=device, + cameras=cameras, + lights=kwargs.get("lights", PointLights()).to(device), + ), + ) + return renderer(meshes) + + def _handle_render_inputs( + self, + model_ids: Optional[List[str]] = None, + categories: Optional[List[str]] = None, + sample_nums: Optional[List[int]] = None, + idxs: Optional[List[int]] = None, + ) -> List[int]: + """ + Helper function for converting user provided model_ids, categories and sample_nums + to indices of models in the loaded dataset. If model idxs are provided, we check if + the idxs are valid. If no models are specified, the first model in the loaded dataset + is chosen. The function returns the file paths to the selected models. + + Args: + model_ids: List[str] of model_ids of models to be rendered. + categories: List[str] of categories to be rendered. + sample_nums: List[int] of number of models to be randomly sampled from + each category. + idxs: List[int] of indices of models to be rendered in the dataset. + + Returns: + List of paths of models to be rendered. + """ + # Get corresponding indices if model_ids are supplied. + if model_ids is not None and len(model_ids) > 0: + idxs = [] + for model_id in model_ids: + if model_id not in self.model_ids: + raise ValueError( + "model_id %s not found in the loaded dataset." % model_id + ) + idxs.append(self.model_ids.index(model_id)) + + # Sample random models if categories and sample_nums are supplied and get + # the corresponding indices. + elif categories is not None and len(categories) > 0: + sample_nums = [1] if sample_nums is None else sample_nums + if len(categories) != len(sample_nums) and len(sample_nums) != 1: + raise ValueError( + "categories and sample_nums needs to be of the same length or " + "sample_nums needs to be of length 1." + ) + + idxs_tensor = torch.empty(0, dtype=torch.int32) + for i in range(len(categories)): + category = self.synset_inv.get(categories[i], categories[i]) + if category not in self.synset_inv.values(): + raise ValueError( + "Category %s is not in the loaded dataset." % category + ) + # Broadcast if sample_nums has length of 1. + sample_num = sample_nums[i] if len(sample_nums) > 1 else sample_nums[0] + sampled_idxs = self._sample_idxs_from_category( + sample_num=sample_num, category=category + ) + # pyre-fixme[6]: For 1st param expected `Union[List[Tensor], + # typing.Tuple[Tensor, ...]]` but got `Tuple[Tensor, List[int]]`. + idxs_tensor = torch.cat((idxs_tensor, sampled_idxs)) + idxs = idxs_tensor.tolist() + # Check if the indices are valid if idxs are supplied. + elif idxs is not None and len(idxs) > 0: + if any(idx < 0 or idx >= len(self.model_ids) for idx in idxs): + raise IndexError( + "One or more idx values are out of bounds. Indices need to be" + "between 0 and %s." % (len(self.model_ids) - 1) + ) + # Check if sample_nums is specified, if so sample sample_nums[0] number + # of indices from the entire loaded dataset. Otherwise randomly select one + # index from the dataset. + else: + sample_nums = [1] if sample_nums is None else sample_nums + if len(sample_nums) > 1: + msg = ( + "More than one sample sizes specified, now sampling " + "%d models from the dataset." % sample_nums[0] + ) + warnings.warn(msg) + idxs = self._sample_idxs_from_category(sample_nums[0]) + return idxs + + def _sample_idxs_from_category( + self, sample_num: int = 1, category: Optional[str] = None + ) -> List[int]: + """ + Helper function for sampling a number of indices from the given category. + + Args: + sample_num: number of indices to be sampled from the given category. + category: category synset of the category to be sampled from. If not + specified, sample from all models in the loaded dataset. + """ + start = self.synset_start_idxs[category] if category is not None else 0 + range_len = ( + self.synset_num_models[category] if category is not None else self.__len__() + ) + replacement = sample_num > range_len + sampled_idxs = ( + torch.multinomial( + torch.ones((range_len), dtype=torch.float32), + sample_num, + replacement=replacement, + ) + + start + ) + if replacement: + msg = ( + "Sample size %d is larger than the number of objects in %s, " + "values sampled with replacement." + ) % ( + sample_num, + "category " + category if category is not None else "all categories", + ) + warnings.warn(msg) + # pyre-fixme[7]: Expected `List[int]` but got `Tensor`. + return sampled_idxs diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/utils.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f651604c8297d98f89850d21353e84f8713b7dff --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/datasets/utils.py @@ -0,0 +1,50 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from typing import Dict, List + +from pytorch3d.renderer.mesh import TexturesAtlas +from pytorch3d.structures import Meshes + + +def collate_batched_meshes(batch: List[Dict]): # pragma: no cover + """ + Take a list of objects in the form of dictionaries and merge them + into a single dictionary. This function can be used with a Dataset + object to create a torch.utils.data.Dataloader which directly + returns Meshes objects. + TODO: Add support for textures. + + Args: + batch: List of dictionaries containing information about objects + in the dataset. + + Returns: + collated_dict: Dictionary of collated lists. If batch contains both + verts and faces, a collated mesh batch is also returned. + """ + if batch is None or len(batch) == 0: + return None + collated_dict = {} + for k in batch[0].keys(): + collated_dict[k] = [d[k] for d in batch] + + collated_dict["mesh"] = None + if {"verts", "faces"}.issubset(collated_dict.keys()): + + textures = None + if "textures" in collated_dict: + textures = TexturesAtlas(atlas=collated_dict["textures"]) + + collated_dict["mesh"] = Meshes( + verts=collated_dict["verts"], + faces=collated_dict["faces"], + textures=textures, + ) + + return collated_dict diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/__init__.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ac1a72bde66f104691245d2de4e83c6863718d5 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/__init__.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ac1a72bde66f104691245d2de4e83c6863718d5 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/blender_dataset_map_provider.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/blender_dataset_map_provider.py new file mode 100644 index 0000000000000000000000000000000000000000..3db89a817295a1ab8691e2d5641d3f3dbf67ecfd --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/blender_dataset_map_provider.py @@ -0,0 +1,55 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +import torch +from pytorch3d.implicitron.tools.config import registry + +from .load_blender import load_blender_data +from .single_sequence_dataset import ( + _interpret_blender_cameras, + SingleSceneDatasetMapProviderBase, +) + + +@registry.register +class BlenderDatasetMapProvider(SingleSceneDatasetMapProviderBase): + """ + Provides data for one scene from Blender synthetic dataset. + Uses the code in load_blender.py + + Members: + base_dir: directory holding the data for the scene. + object_name: The name of the scene (e.g. "lego"). This is just used as a label. + It will typically be equal to the name of the directory self.base_dir. + path_manager_factory: Creates path manager which may be used for + interpreting paths. + n_known_frames_for_test: If set, training frames are included in the val + and test datasets, and this many random training frames are added to + each test batch. If not set, test batches each contain just a single + testing frame. + """ + + def _load_data(self) -> None: + path_manager = self.path_manager_factory.get() + images, poses, _, hwf, i_split = load_blender_data( + self.base_dir, + testskip=1, + path_manager=path_manager, + ) + H, W, focal = hwf + images_masks = torch.from_numpy(images).permute(0, 3, 1, 2) + + # pyre-ignore[16] + self.poses = _interpret_blender_cameras(poses, focal) + # pyre-ignore[16] + self.images = images_masks[:, :3] + # pyre-ignore[16] + self.fg_probabilities = images_masks[:, 3:4] + # pyre-ignore[16] + self.i_split = i_split diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/data_loader_map_provider.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/data_loader_map_provider.py new file mode 100644 index 0000000000000000000000000000000000000000..1e33887b7b295630adeebb200f75fe2a5db4a6df --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/data_loader_map_provider.py @@ -0,0 +1,526 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from dataclasses import dataclass +from enum import Enum +from typing import Iterator, List, Optional, Tuple + +import torch +from pytorch3d.implicitron.tools.config import registry, ReplaceableBase +from torch.utils.data import ( + BatchSampler, + ConcatDataset, + DataLoader, + RandomSampler, + Sampler, +) + +from .dataset_base import DatasetBase +from .dataset_map_provider import DatasetMap +from .frame_data import FrameData +from .scene_batch_sampler import SceneBatchSampler +from .utils import is_known_frame_scalar + + +@dataclass +class DataLoaderMap: + """ + A collection of data loaders for Implicitron. + + Members: + + train: a data loader for training + val: a data loader for validating during training + test: a data loader for final evaluation + """ + + train: Optional[DataLoader[FrameData]] + val: Optional[DataLoader[FrameData]] + test: Optional[DataLoader[FrameData]] + + def __getitem__(self, split: str) -> Optional[DataLoader[FrameData]]: + """ + Get one of the data loaders by key (name of data split) + """ + if split not in ["train", "val", "test"]: + raise ValueError(f"{split} was not a valid split name (train/val/test)") + return getattr(self, split) + + +class DataLoaderMapProviderBase(ReplaceableBase): + """ + Provider of a collection of data loaders for a given collection of datasets. + """ + + def get_data_loader_map(self, datasets: DatasetMap) -> DataLoaderMap: + """ + Returns a collection of data loaders for a given collection of datasets. + """ + raise NotImplementedError() + + +@registry.register +class SimpleDataLoaderMapProvider(DataLoaderMapProviderBase): + """ + Trivial implementation of DataLoaderMapProviderBase. + + If a dataset returns batches from get_eval_batches(), then + they will be what the corresponding dataloader returns, + independently of any of the fields on this class. + + Otherwise, returns shuffled batches. + """ + + batch_size: int = 1 + num_workers: int = 0 + dataset_length_train: int = 0 + dataset_length_val: int = 0 + dataset_length_test: int = 0 + + def get_data_loader_map(self, datasets: DatasetMap) -> DataLoaderMap: + """ + Returns a collection of data loaders for a given collection of datasets. + """ + return DataLoaderMap( + train=self._make_data_loader( + datasets.train, + self.dataset_length_train, + ), + val=self._make_data_loader( + datasets.val, + self.dataset_length_val, + ), + test=self._make_data_loader( + datasets.test, + self.dataset_length_test, + ), + ) + + def _make_data_loader( + self, + dataset: Optional[DatasetBase], + num_batches: int, + ) -> Optional[DataLoader[FrameData]]: + """ + Returns the dataloader for a dataset. + + Args: + dataset: the dataset + num_batches: possible ceiling on number of batches per epoch + """ + if dataset is None: + return None + + data_loader_kwargs = { + "num_workers": self.num_workers, + "collate_fn": dataset.frame_data_type.collate, + } + + eval_batches = dataset.get_eval_batches() + if eval_batches is not None: + return DataLoader( + dataset, + batch_sampler=eval_batches, + **data_loader_kwargs, + ) + + if num_batches > 0: + num_samples = self.batch_size * num_batches + else: + num_samples = None + + # sample with replacement only if a custom number of samples is specified + sampler = RandomSampler( + dataset, + replacement=num_samples is not None, + num_samples=num_samples, + ) + + batch_sampler = BatchSampler(sampler, self.batch_size, drop_last=True) + return DataLoader( + dataset, + batch_sampler=batch_sampler, + **data_loader_kwargs, + ) + + +class DoublePoolBatchSampler(Sampler[List[int]]): + """ + Batch sampler for making random batches of a single frame + from one list and a number of known frames from another list. + """ + + def __init__( + self, + first_indices: List[int], + rest_indices: List[int], + batch_size: int, + replacement: bool, + num_batches: Optional[int] = None, + ) -> None: + """ + Args: + first_indices: indexes of dataset items to use as the first element + of each batch. + rest_indices: indexes of dataset items to use as the subsequent + elements of each batch. Not used if batch_size==1. + batch_size: The common size of any batch. + replacement: Whether the sampling of first items is with replacement. + num_batches: The number of batches in an epoch. If 0 or None, + one epoch is the length of `first_indices`. + """ + self.first_indices = first_indices + self.rest_indices = rest_indices + self.batch_size = batch_size + self.replacement = replacement + self.num_batches = None if num_batches == 0 else num_batches + + if batch_size - 1 > len(rest_indices): + raise ValueError( + f"Cannot make up ({batch_size})-batches from {len(self.rest_indices)}" + ) + + # copied from RandomSampler + seed = int(torch.empty((), dtype=torch.int64).random_().item()) + self.generator = torch.Generator() + self.generator.manual_seed(seed) + + def __len__(self) -> int: + if self.num_batches is not None: + return self.num_batches + return len(self.first_indices) + + def __iter__(self) -> Iterator[List[int]]: + num_batches = self.num_batches + if self.replacement: + i_first = torch.randint( + len(self.first_indices), + size=(len(self),), + generator=self.generator, + ) + elif num_batches is not None: + n_copies = 1 + (num_batches - 1) // len(self.first_indices) + raw_indices = [ + torch.randperm(len(self.first_indices), generator=self.generator) + for _ in range(n_copies) + ] + i_first = torch.cat(raw_indices)[:num_batches] + else: + i_first = torch.randperm(len(self.first_indices), generator=self.generator) + first_indices = [self.first_indices[i] for i in i_first] + + if self.batch_size == 1: + for first_index in first_indices: + yield [first_index] + return + + for first_index in first_indices: + # Consider using this class in a program which sets the seed. This use + # of randperm means that rerunning with a higher batch_size + # results in batches whose first elements as the first run. + i_rest = torch.randperm( + len(self.rest_indices), + generator=self.generator, + )[: self.batch_size - 1] + yield [first_index] + [self.rest_indices[i] for i in i_rest] + + +class BatchConditioningType(Enum): + """ + Ways to add conditioning frames for the val and test batches. + + SAME: Use the corresponding dataset for all elements of val batches + without regard to frame type. + TRAIN: Use the corresponding dataset for the first element of each + batch, and the training dataset for the extra conditioning + elements. No regard to frame type. + KNOWN: Use frames from the corresponding dataset but separate them + according to their frame_type. Each batch will contain one UNSEEN + frame followed by many KNOWN frames. + """ + + SAME = "same" + TRAIN = "train" + KNOWN = "known" + + +@registry.register +class SequenceDataLoaderMapProvider(DataLoaderMapProviderBase): + """ + Default implementation of DataLoaderMapProviderBase. + + If a dataset returns batches from get_eval_batches(), then + they will be what the corresponding dataloader returns, + independently of any of the fields on this class. + + If conditioning is not required, then the batch size should + be set as 1, and most of the fields do not matter. + + If conditioning is required, each batch will contain one main + frame first to predict and the, rest of the elements are for + conditioning. + + If images_per_seq_options is left empty, the conditioning + frames are picked according to the conditioning type given. + This does not have regard to the order of frames in a + scene, or which frames belong to what scene. + + If images_per_seq_options is given, then the conditioning types + must be SAME and the remaining fields are used. + + Members: + batch_size: The size of the batch of the data loader. + num_workers: Number of data-loading threads in each data loader. + dataset_length_train: The number of batches in a training epoch. Or 0 to mean + an epoch is the length of the training set. + dataset_length_val: The number of batches in a validation epoch. Or 0 to mean + an epoch is the length of the validation set. + dataset_length_test: The number of batches in a testing epoch. Or 0 to mean + an epoch is the length of the test set. + train_conditioning_type: Whether the train data loader should use + only known frames for conditioning. + Only used if batch_size>1 and train dataset is + present and does not return eval_batches. + val_conditioning_type: Whether the val data loader should use + training frames or known frames for conditioning. + Only used if batch_size>1 and val dataset is + present and does not return eval_batches. + test_conditioning_type: Whether the test data loader should use + training frames or known frames for conditioning. + Only used if batch_size>1 and test dataset is + present and does not return eval_batches. + images_per_seq_options: Possible numbers of frames sampled per sequence in a batch. + If a conditioning_type is KNOWN or TRAIN, then this must be left at its initial + value. Empty (the default) means that we are not careful about which frames + come from which scene. + sample_consecutive_frames: if True, will sample a contiguous interval of frames + in the sequence. It first sorts the frames by timestimps when available, + otherwise by frame numbers, finds the connected segments within the sequence + of sufficient length, then samples a random pivot element among them and + ideally uses it as a middle of the temporal window, shifting the borders + where necessary. This strategy mitigates the bias against shorter segments + and their boundaries. + consecutive_frames_max_gap: if a number > 0, then used to define the maximum + difference in frame_number of neighbouring frames when forming connected + segments; if both this and consecutive_frames_max_gap_seconds are 0s, + the whole sequence is considered a segment regardless of frame numbers. + consecutive_frames_max_gap_seconds: if a number > 0.0, then used to define the + maximum difference in frame_timestamp of neighbouring frames when forming + connected segments; if both this and consecutive_frames_max_gap are 0s, + the whole sequence is considered a segment regardless of frame timestamps. + """ + + batch_size: int = 1 + num_workers: int = 0 + dataset_length_train: int = 0 + dataset_length_val: int = 0 + dataset_length_test: int = 0 + train_conditioning_type: BatchConditioningType = BatchConditioningType.SAME + val_conditioning_type: BatchConditioningType = BatchConditioningType.SAME + test_conditioning_type: BatchConditioningType = BatchConditioningType.KNOWN + images_per_seq_options: Tuple[int, ...] = () + sample_consecutive_frames: bool = False + consecutive_frames_max_gap: int = 0 + consecutive_frames_max_gap_seconds: float = 0.1 + + def get_data_loader_map(self, datasets: DatasetMap) -> DataLoaderMap: + """ + Returns a collection of data loaders for a given collection of datasets. + """ + return DataLoaderMap( + train=self._make_data_loader( + datasets.train, + self.dataset_length_train, + datasets.train, + self.train_conditioning_type, + ), + val=self._make_data_loader( + datasets.val, + self.dataset_length_val, + datasets.train, + self.val_conditioning_type, + ), + test=self._make_data_loader( + datasets.test, + self.dataset_length_test, + datasets.train, + self.test_conditioning_type, + ), + ) + + def _make_data_loader( + self, + dataset: Optional[DatasetBase], + num_batches: int, + train_dataset: Optional[DatasetBase], + conditioning_type: BatchConditioningType, + ) -> Optional[DataLoader[FrameData]]: + """ + Returns the dataloader for a dataset. + + Args: + dataset: the dataset + num_batches: possible ceiling on number of batches per epoch + train_dataset: the training dataset, used if conditioning_type==TRAIN + conditioning_type: source for padding of batches + """ + if dataset is None: + return None + + data_loader_kwargs = { + "num_workers": self.num_workers, + "collate_fn": dataset.frame_data_type.collate, + } + + eval_batches = dataset.get_eval_batches() + if eval_batches is not None: + return DataLoader( + dataset, + batch_sampler=eval_batches, + **data_loader_kwargs, + ) + + scenes_matter = len(self.images_per_seq_options) > 0 + if scenes_matter and conditioning_type != BatchConditioningType.SAME: + raise ValueError( + f"{conditioning_type} cannot be used with images_per_seq " + + str(self.images_per_seq_options) + ) + + if self.batch_size == 1 or ( + not scenes_matter and conditioning_type == BatchConditioningType.SAME + ): + return self._simple_loader(dataset, num_batches, data_loader_kwargs) + + if scenes_matter: + assert conditioning_type == BatchConditioningType.SAME + batch_sampler = SceneBatchSampler( + dataset, + self.batch_size, + num_batches=len(dataset) if num_batches <= 0 else num_batches, + images_per_seq_options=self.images_per_seq_options, + sample_consecutive_frames=self.sample_consecutive_frames, + consecutive_frames_max_gap=self.consecutive_frames_max_gap, + consecutive_frames_max_gap_seconds=self.consecutive_frames_max_gap_seconds, + ) + return DataLoader( + dataset, + batch_sampler=batch_sampler, + **data_loader_kwargs, + ) + + if conditioning_type == BatchConditioningType.TRAIN: + return self._train_loader( + dataset, train_dataset, num_batches, data_loader_kwargs + ) + + assert conditioning_type == BatchConditioningType.KNOWN + return self._known_loader(dataset, num_batches, data_loader_kwargs) + + def _simple_loader( + self, + dataset: DatasetBase, + num_batches: int, + data_loader_kwargs: dict, + ) -> DataLoader[FrameData]: + """ + Return a simple loader for frames in the dataset. + + This is equivalent to + Dataloader(dataset, batch_size=self.batch_size, **data_loader_kwargs) + except that num_batches is fixed. + + Args: + dataset: the dataset + num_batches: possible ceiling on number of batches per epoch + data_loader_kwargs: common args for dataloader + """ + if num_batches > 0: + num_samples = self.batch_size * num_batches + replacement = True + else: + num_samples = None + replacement = False + sampler = RandomSampler( + dataset, replacement=replacement, num_samples=num_samples + ) + batch_sampler = BatchSampler(sampler, self.batch_size, drop_last=True) + return DataLoader( + dataset, + batch_sampler=batch_sampler, + **data_loader_kwargs, + ) + + def _train_loader( + self, + dataset: DatasetBase, + train_dataset: Optional[DatasetBase], + num_batches: int, + data_loader_kwargs: dict, + ) -> DataLoader[FrameData]: + """ + Return the loader for TRAIN conditioning. + + Args: + dataset: the dataset + train_dataset: the training dataset + num_batches: possible ceiling on number of batches per epoch + data_loader_kwargs: common args for dataloader + """ + if train_dataset is None: + raise ValueError("No training data for conditioning.") + length = len(dataset) + first_indices = list(range(length)) + rest_indices = list(range(length, length + len(train_dataset))) + sampler = DoublePoolBatchSampler( + first_indices=first_indices, + rest_indices=rest_indices, + batch_size=self.batch_size, + replacement=True, + num_batches=num_batches, + ) + return DataLoader( + ConcatDataset([dataset, train_dataset]), + batch_sampler=sampler, + **data_loader_kwargs, + ) + + def _known_loader( + self, + dataset: DatasetBase, + num_batches: int, + data_loader_kwargs: dict, + ) -> DataLoader[FrameData]: + """ + Return the loader for KNOWN conditioning. + + Args: + dataset: the dataset + num_batches: possible ceiling on number of batches per epoch + data_loader_kwargs: common args for dataloader + """ + first_indices, rest_indices = [], [] + for idx in range(len(dataset)): + frame_type = dataset[idx].frame_type + assert isinstance(frame_type, str) + if is_known_frame_scalar(frame_type): + rest_indices.append(idx) + else: + first_indices.append(idx) + sampler = DoublePoolBatchSampler( + first_indices=first_indices, + rest_indices=rest_indices, + batch_size=self.batch_size, + replacement=True, + num_batches=num_batches, + ) + return DataLoader( + dataset, + batch_sampler=sampler, + **data_loader_kwargs, + ) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/data_source.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/data_source.py new file mode 100644 index 0000000000000000000000000000000000000000..5c789e7da142401fd3fd1ee7505b666ab10574d1 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/data_source.py @@ -0,0 +1,108 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from typing import Optional, Tuple + +from pytorch3d.implicitron.tools.config import ( + registry, + ReplaceableBase, + run_auto_creation, +) +from pytorch3d.renderer.cameras import CamerasBase + +from .data_loader_map_provider import DataLoaderMap, DataLoaderMapProviderBase +from .dataset_map_provider import DatasetMap, DatasetMapProviderBase + + +class DataSourceBase(ReplaceableBase): + """ + Base class for a data source in Implicitron. It encapsulates Dataset + and DataLoader configuration. + """ + + def get_datasets_and_dataloaders(self) -> Tuple[DatasetMap, DataLoaderMap]: + raise NotImplementedError() + + @property + def all_train_cameras(self) -> Optional[CamerasBase]: + """ + DEPRECATED! The property will be removed in future versions. + If the data is all for a single scene, a list + of the known training cameras for that scene, which is + used for evaluating the viewpoint difficulty of the + unseen cameras. + """ + raise NotImplementedError() + + +@registry.register +class ImplicitronDataSource(DataSourceBase): # pyre-ignore[13] + """ + Represents the data used in Implicitron. This is the only implementation + of DataSourceBase provided. + + Members: + dataset_map_provider_class_type: identifies type for dataset_map_provider. + e.g. JsonIndexDatasetMapProvider for Co3D. + data_loader_map_provider_class_type: identifies type for data_loader_map_provider. + """ + + dataset_map_provider: DatasetMapProviderBase + dataset_map_provider_class_type: str + data_loader_map_provider: DataLoaderMapProviderBase + data_loader_map_provider_class_type: str = "SequenceDataLoaderMapProvider" + + @classmethod + def pre_expand(cls) -> None: + # use try/finally to bypass cinder's lazy imports + try: + from .blender_dataset_map_provider import ( # noqa: F401 + BlenderDatasetMapProvider, + ) + from .json_index_dataset_map_provider import ( # noqa: F401 + JsonIndexDatasetMapProvider, + ) + from .json_index_dataset_map_provider_v2 import ( # noqa: F401 + JsonIndexDatasetMapProviderV2, + ) + from .llff_dataset_map_provider import LlffDatasetMapProvider # noqa: F401 + from .rendered_mesh_dataset_map_provider import ( # noqa: F401 + RenderedMeshDatasetMapProvider, + ) + from .train_eval_data_loader_provider import ( # noqa: F401 + TrainEvalDataLoaderMapProvider, + ) + + try: + from .sql_dataset_provider import ( # noqa: F401 # pyre-ignore + SqlIndexDatasetMapProvider, + ) + except ModuleNotFoundError: + pass # environment without SQL dataset + finally: + pass + + def __post_init__(self): + run_auto_creation(self) + self._all_train_cameras_cache: Optional[Tuple[Optional[CamerasBase]]] = None + + def get_datasets_and_dataloaders(self) -> Tuple[DatasetMap, DataLoaderMap]: + datasets = self.dataset_map_provider.get_dataset_map() + dataloaders = self.data_loader_map_provider.get_data_loader_map(datasets) + return datasets, dataloaders + + @property + def all_train_cameras(self) -> Optional[CamerasBase]: + """ + DEPRECATED! The property will be removed in future versions. + """ + if self._all_train_cameras_cache is None: # pyre-ignore[16] + all_train_cameras = self.dataset_map_provider.get_all_train_cameras() + self._all_train_cameras_cache = (all_train_cameras,) + + return self._all_train_cameras_cache[0] diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/dataset_base.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/dataset_base.py new file mode 100644 index 0000000000000000000000000000000000000000..4b0827882c82b1a0e838e5ed77e9e3d8559ac2bf --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/dataset_base.py @@ -0,0 +1,147 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from collections import defaultdict +from dataclasses import dataclass +from typing import ( + ClassVar, + Dict, + Iterable, + Iterator, + List, + Optional, + Sequence, + Tuple, + Type, +) + +import torch + +from pytorch3d.implicitron.dataset.frame_data import FrameData +from pytorch3d.implicitron.dataset.utils import GenericWorkaround + + +@dataclass(eq=False) +class DatasetBase(GenericWorkaround, torch.utils.data.Dataset[FrameData]): + """ + Base class to describe a dataset to be used with Implicitron. + + The dataset is made up of frames, and the frames are grouped into sequences. + Each sequence has a name (a string). + (A sequence could be a video, or a set of images of one scene.) + + This means they have a __getitem__ which returns an instance of a FrameData, + which will describe one frame in one sequence. + """ + + # _seq_to_idx is a member which implementations can define. + # It maps sequence name to the sequence's global frame indices. + # It is used for the default implementations of some functions in this class. + # Implementations which override them are free to ignore it. + # _seq_to_idx: Dict[str, List[int]] = field(init=False) + + def __len__(self) -> int: + raise NotImplementedError() + + def get_frame_numbers_and_timestamps( + self, idxs: Sequence[int], subset_filter: Optional[Sequence[str]] = None + ) -> List[Tuple[int, float]]: + """ + If the sequences in the dataset are videos rather than + unordered views, then the dataset should override this method to + return the index and timestamp in their videos of the frames whose + indices are given in `idxs`. In addition, + the values in _seq_to_idx should be in ascending order. + If timestamps are absent, they should be replaced with a constant. + + This is used for letting SceneBatchSampler identify consecutive + frames. + + Args: + idxs: frame index in self + subset_filter: If given, an index in idxs is ignored if the + corresponding frame is not in any of the named subsets. + + Returns: + tuple of + - frame index in video + - timestamp of frame in video + """ + raise ValueError("This dataset does not contain videos.") + + def join(self, other_datasets: Iterable["DatasetBase"]) -> None: + """ + Joins the current dataset with a list of other datasets of the same type. + """ + raise NotImplementedError() + + def get_eval_batches(self) -> Optional[List[List[int]]]: + return None + + def sequence_names(self) -> Iterable[str]: + """Returns an iterator over sequence names in the dataset.""" + # pyre-ignore[16] + return self._seq_to_idx.keys() + + def category_to_sequence_names(self) -> Dict[str, List[str]]: + """ + Returns a dict mapping from each dataset category to a list of its + sequence names. + + Returns: + category_to_sequence_names: Dict {category_i: [..., sequence_name_j, ...]} + """ + c2seq = defaultdict(list) + for sequence_name in self.sequence_names(): + first_frame_idx = next(self.sequence_indices_in_order(sequence_name)) + # crashes without overriding __getitem__ + sequence_category = self[first_frame_idx].sequence_category + c2seq[sequence_category].append(sequence_name) + return dict(c2seq) + + def sequence_frames_in_order( + self, seq_name: str, subset_filter: Optional[Sequence[str]] = None + ) -> Iterator[Tuple[float, int, int]]: + """Returns an iterator over the frame indices in a given sequence. + We attempt to first sort by timestamp (if they are available), + then by frame number. + + Args: + seq_name: the name of the sequence. + + Returns: + an iterator over triplets `(timestamp, frame_no, dataset_idx)`, + where `frame_no` is the index within the sequence, and + `dataset_idx` is the index within the dataset. + `None` timestamps are replaced with 0s. + """ + # pyre-ignore[16] + seq_frame_indices = self._seq_to_idx[seq_name] + nos_timestamps = self.get_frame_numbers_and_timestamps( + seq_frame_indices, subset_filter + ) + + yield from sorted( + [ + (timestamp, frame_no, idx) + for idx, (frame_no, timestamp) in zip(seq_frame_indices, nos_timestamps) + ] + ) + + def sequence_indices_in_order( + self, seq_name: str, subset_filter: Optional[Sequence[str]] = None + ) -> Iterator[int]: + """Same as `sequence_frames_in_order` but returns the iterator over + only dataset indices. + """ + for _, _, idx in self.sequence_frames_in_order(seq_name, subset_filter): + yield idx + + # frame_data_type is the actual type of frames returned by the dataset. + # Collation uses its classmethod `collate` + frame_data_type: ClassVar[Type[FrameData]] = FrameData diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/dataset_map_provider.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/dataset_map_provider.py new file mode 100644 index 0000000000000000000000000000000000000000..7133b36421efce0051cec6ecf9a9c5d34cacdaa8 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/dataset_map_provider.py @@ -0,0 +1,142 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import logging +import os +from dataclasses import dataclass +from typing import Iterable, Iterator, Optional + +from iopath.common.file_io import PathManager +from pytorch3d.implicitron.tools.config import registry, ReplaceableBase +from pytorch3d.renderer.cameras import CamerasBase + +from .dataset_base import DatasetBase + + +@dataclass +class DatasetMap: + """ + A collection of datasets for implicitron. + + Members: + + train: a dataset for training + val: a dataset for validating during training + test: a dataset for final evaluation + """ + + train: Optional[DatasetBase] + val: Optional[DatasetBase] + test: Optional[DatasetBase] + + def __getitem__(self, split: str) -> Optional[DatasetBase]: + """ + Get one of the datasets by key (name of data split) + """ + if split not in ["train", "val", "test"]: + raise ValueError(f"{split} was not a valid split name (train/val/test)") + return getattr(self, split) + + def iter_datasets(self) -> Iterator[DatasetBase]: + """ + Iterator over all datasets. + """ + if self.train is not None: + yield self.train + if self.val is not None: + yield self.val + if self.test is not None: + yield self.test + + def join(self, other_dataset_maps: Iterable["DatasetMap"]) -> None: + """ + Joins the current DatasetMap with other dataset maps from the input list. + + For each subset of each dataset map (train/val/test), the function + omits joining the subsets that are None. + + Note the train/val/test datasets of the current dataset map will be + modified in-place. + + Args: + other_dataset_maps: The list of dataset maps to be joined into the + current dataset map. + """ + for set_ in ["train", "val", "test"]: + dataset_list = [ + getattr(self, set_), + *[getattr(dmap, set_) for dmap in other_dataset_maps], + ] + dataset_list = [d for d in dataset_list if d is not None] + if len(dataset_list) == 0: + setattr(self, set_, None) + continue + d0 = dataset_list[0] + if len(dataset_list) > 1: + d0.join(dataset_list[1:]) + setattr(self, set_, d0) + + +class DatasetMapProviderBase(ReplaceableBase): + """ + Base class for a provider of training / validation and testing + dataset objects. + """ + + def get_dataset_map(self) -> DatasetMap: + """ + Returns: + An object containing the torch.Dataset objects in train/val/test fields. + """ + raise NotImplementedError() + + def get_all_train_cameras(self) -> Optional[CamerasBase]: + """ + DEPRECATED! The function will be removed in future versions. + If the data is all for a single scene, returns a list + of the known training cameras for that scene, which is + used for evaluating the difficulty of the unknown + cameras. Otherwise return None. + """ + raise NotImplementedError() + + +@registry.register +class PathManagerFactory(ReplaceableBase): + """ + Base class and default implementation of a tool which dataset_map_provider implementations + may use to construct a path manager if needed. + + Args: + silence_logs: Whether to reduce log output from iopath library. + """ + + silence_logs: bool = True + + def get(self) -> Optional[PathManager]: + """ + Makes a PathManager if needed. + For open source users, this function should always return None. + Internally, this allows manifold access. + """ + if os.environ.get("INSIDE_RE_WORKER", False): + return None + + try: + from iopath.fb.manifold import ManifoldPathHandler + except ImportError: + return None + + if self.silence_logs: + logging.getLogger("iopath.fb.manifold").setLevel(logging.CRITICAL) + logging.getLogger("iopath.common.file_io").setLevel(logging.CRITICAL) + + path_manager = PathManager() + path_manager.register_handler(ManifoldPathHandler()) + + return path_manager diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/frame_data.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/frame_data.py new file mode 100644 index 0000000000000000000000000000000000000000..631bea8a945d0f27bf482ae32ed93e99a1fa41bb --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/frame_data.py @@ -0,0 +1,780 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import os +from abc import ABC, abstractmethod +from collections import defaultdict +from dataclasses import dataclass, field, fields +from typing import ( + Any, + ClassVar, + Generic, + List, + Mapping, + Optional, + Tuple, + Type, + TypeVar, + Union, +) + +import numpy as np +import torch + +from pytorch3d.implicitron.dataset import types +from pytorch3d.implicitron.dataset.utils import ( + adjust_camera_to_bbox_crop_, + adjust_camera_to_image_scale_, + bbox_xyxy_to_xywh, + clamp_box_to_image_bounds_and_round, + crop_around_box, + GenericWorkaround, + get_bbox_from_mask, + get_clamp_bbox, + load_depth, + load_depth_mask, + load_image, + load_mask, + load_pointcloud, + rescale_bbox, + resize_image, + safe_as_tensor, +) +from pytorch3d.implicitron.tools.config import registry, ReplaceableBase +from pytorch3d.renderer.camera_utils import join_cameras_as_batch +from pytorch3d.renderer.cameras import CamerasBase, PerspectiveCameras +from pytorch3d.structures.pointclouds import join_pointclouds_as_batch, Pointclouds + + +@dataclass +class FrameData(Mapping[str, Any]): + """ + A type of the elements returned by indexing the dataset object. + It can represent both individual frames and batches of thereof; + in this documentation, the sizes of tensors refer to single frames; + add the first batch dimension for the collation result. + + Args: + frame_number: The number of the frame within its sequence. + 0-based continuous integers. + sequence_name: The unique name of the frame's sequence. + sequence_category: The object category of the sequence. + frame_timestamp: The time elapsed since the start of a sequence in sec. + image_size_hw: The size of the original image in pixels; (height, width) + tensor of shape (2,). Note that it is optional, e.g. it can be `None` + if the frame annotation has no size ans image_rgb has not [yet] been + loaded. Image-less FrameData is valid but mutators like crop/resize + may fail if the original image size cannot be deduced. + effective_image_size_hw: The size of the image after mutations such as + crop/resize in pixels; (height, width). if the image has not been mutated, + it is equal to `image_size_hw`. Note that it is also optional, for the + same reason as `image_size_hw`. + image_path: The qualified path to the loaded image (with dataset_root). + image_rgb: A Tensor of shape `(3, H, W)` holding the RGB image + of the frame; elements are floats in [0, 1]. + mask_crop: A binary mask of shape `(1, H, W)` denoting the valid image + regions. Regions can be invalid (mask_crop[i,j]=0) in case they + are a result of zero-padding of the image after cropping around + the object bounding box; elements are floats in {0.0, 1.0}. + depth_path: The qualified path to the frame's depth map. + depth_map: A float Tensor of shape `(1, H, W)` holding the depth map + of the frame; values correspond to distances from the camera; + use `depth_mask` and `mask_crop` to filter for valid pixels. + depth_mask: A binary mask of shape `(1, H, W)` denoting pixels of the + depth map that are valid for evaluation, they have been checked for + consistency across views; elements are floats in {0.0, 1.0}. + mask_path: A qualified path to the foreground probability mask. + fg_probability: A Tensor of `(1, H, W)` denoting the probability of the + pixels belonging to the captured object; elements are floats + in [0, 1]. + bbox_xywh: The bounding box tightly enclosing the foreground object in the + format (x0, y0, width, height). The convention assumes that + `x0+width` and `y0+height` includes the boundary of the box. + I.e., to slice out the corresponding crop from an image tensor `I` + we execute `crop = I[..., y0:y0+height, x0:x0+width]` + crop_bbox_xywh: The bounding box denoting the boundaries of `image_rgb` + in the original image coordinates in the format (x0, y0, width, height). + The convention is the same as for `bbox_xywh`. `crop_bbox_xywh` differs + from `bbox_xywh` due to padding (which can happen e.g. due to + setting `JsonIndexDataset.box_crop_context > 0`) + camera: A PyTorch3D camera object corresponding the frame's viewpoint, + corrected for cropping if it happened. + camera_quality_score: The score proportional to the confidence of the + frame's camera estimation (the higher the more accurate). + point_cloud_quality_score: The score proportional to the accuracy of the + frame's sequence point cloud (the higher the more accurate). + sequence_point_cloud_path: The path to the sequence's point cloud. + sequence_point_cloud: A PyTorch3D Pointclouds object holding the + point cloud corresponding to the frame's sequence. When the object + represents a batch of frames, point clouds may be deduplicated; + see `sequence_point_cloud_idx`. + sequence_point_cloud_idx: Integer indices mapping frame indices to the + corresponding point clouds in `sequence_point_cloud`; to get the + corresponding point cloud to `image_rgb[i]`, use + `sequence_point_cloud[sequence_point_cloud_idx[i]]`. + frame_type: The type of the loaded frame specified in + `subset_lists_file`, if provided. + meta: A dict for storing additional frame information. + """ + + frame_number: Optional[torch.LongTensor] + sequence_name: Union[str, List[str]] + sequence_category: Union[str, List[str]] + frame_timestamp: Optional[torch.Tensor] = None + image_size_hw: Optional[torch.LongTensor] = None + effective_image_size_hw: Optional[torch.LongTensor] = None + image_path: Union[str, List[str], None] = None + image_rgb: Optional[torch.Tensor] = None + # masks out padding added due to cropping the square bit + mask_crop: Optional[torch.Tensor] = None + depth_path: Union[str, List[str], None] = None + depth_map: Optional[torch.Tensor] = None + depth_mask: Optional[torch.Tensor] = None + mask_path: Union[str, List[str], None] = None + fg_probability: Optional[torch.Tensor] = None + bbox_xywh: Optional[torch.Tensor] = None + crop_bbox_xywh: Optional[torch.Tensor] = None + camera: Optional[PerspectiveCameras] = None + camera_quality_score: Optional[torch.Tensor] = None + point_cloud_quality_score: Optional[torch.Tensor] = None + sequence_point_cloud_path: Union[str, List[str], None] = None + sequence_point_cloud: Optional[Pointclouds] = None + sequence_point_cloud_idx: Optional[torch.Tensor] = None + frame_type: Union[str, List[str], None] = None # known | unseen + meta: dict = field(default_factory=lambda: {}) + + # NOTE that batching resets this attribute + _uncropped: bool = field(init=False, default=True) + + def to(self, *args, **kwargs): + new_params = {} + for field_name in iter(self): + value = getattr(self, field_name) + if isinstance(value, (torch.Tensor, Pointclouds, CamerasBase)): + new_params[field_name] = value.to(*args, **kwargs) + else: + new_params[field_name] = value + frame_data = type(self)(**new_params) + frame_data._uncropped = self._uncropped + return frame_data + + def cpu(self): + return self.to(device=torch.device("cpu")) + + def cuda(self): + return self.to(device=torch.device("cuda")) + + # the following functions make sure **frame_data can be passed to functions + def __iter__(self): + for f in fields(self): + if f.name.startswith("_"): + continue + + yield f.name + + def __getitem__(self, key): + return getattr(self, key) + + def __len__(self): + return sum(1 for f in iter(self)) + + def crop_by_metadata_bbox_( + self, + box_crop_context: float, + ) -> None: + """Crops the frame data in-place by (possibly expanded) bounding box. + The bounding box is taken from the object state (usually taken from + the frame annotation or estimated from the foregroubnd mask). + If the expanded bounding box does not fit the image, it is clamped, + i.e. the image is *not* padded. + + Args: + box_crop_context: rate of expansion for bbox; 0 means no expansion, + + Raises: + ValueError: If the object does not contain a bounding box (usually when no + mask annotation is provided) + ValueError: If the frame data have been cropped or resized, thus the intrinsic + bounding box is not valid for the current image size. + ValueError: If the frame does not have an image size (usually a corner case + when no image has been loaded) + """ + if self.bbox_xywh is None: + raise ValueError( + "Attempted cropping by metadata with empty bounding box. Consider either" + " to remove_empty_masks or turn off box_crop in the dataset config." + ) + + if not self._uncropped: + raise ValueError( + "Trying to apply the metadata bounding box to already cropped " + "or resized image; coordinates have changed." + ) + + self._crop_by_bbox_( + box_crop_context, + self.bbox_xywh, + ) + + def crop_by_given_bbox_( + self, + box_crop_context: float, + bbox_xywh: torch.Tensor, + ) -> None: + """Crops the frame data in-place by (possibly expanded) bounding box. + If the expanded bounding box does not fit the image, it is clamped, + i.e. the image is *not* padded. + + Args: + box_crop_context: rate of expansion for bbox; 0 means no expansion, + bbox_xywh: bounding box in [x0, y0, width, height] format. If float + tensor, values are floored (after converting to [x0, y0, x1, y1]). + + Raises: + ValueError: If the frame does not have an image size (usually a corner case + when no image has been loaded) + """ + self._crop_by_bbox_( + box_crop_context, + bbox_xywh, + ) + + def _crop_by_bbox_( + self, + box_crop_context: float, + bbox_xywh: torch.Tensor, + ) -> None: + """Crops the frame data in-place by (possibly expanded) bounding box. + If the expanded bounding box does not fit the image, it is clamped, + i.e. the image is *not* padded. + + Args: + box_crop_context: rate of expansion for bbox; 0 means no expansion, + bbox_xywh: bounding box in [x0, y0, width, height] format. If float + tensor, values are floored (after converting to [x0, y0, x1, y1]). + + Raises: + ValueError: If the frame does not have an image size (usually a corner case + when no image has been loaded) + """ + effective_image_size_hw = self.effective_image_size_hw + if effective_image_size_hw is None: + raise ValueError("Calling crop on image-less FrameData") + + bbox_xyxy = get_clamp_bbox( + bbox_xywh, + image_path=self.image_path, # pyre-ignore + box_crop_context=box_crop_context, + ) + clamp_bbox_xyxy = clamp_box_to_image_bounds_and_round( + bbox_xyxy, + image_size_hw=tuple(self.effective_image_size_hw), # pyre-ignore + ) + crop_bbox_xywh = bbox_xyxy_to_xywh(clamp_bbox_xyxy) + self.crop_bbox_xywh = crop_bbox_xywh + + if self.fg_probability is not None: + self.fg_probability = crop_around_box( + self.fg_probability, + clamp_bbox_xyxy, + self.mask_path, # pyre-ignore + ) + if self.image_rgb is not None: + self.image_rgb = crop_around_box( + self.image_rgb, + clamp_bbox_xyxy, + self.image_path, # pyre-ignore + ) + + depth_map = self.depth_map + if depth_map is not None: + clamp_bbox_xyxy_depth = rescale_bbox( + clamp_bbox_xyxy, tuple(depth_map.shape[-2:]), effective_image_size_hw + ).long() + self.depth_map = crop_around_box( + depth_map, + clamp_bbox_xyxy_depth, + self.depth_path, # pyre-ignore + ) + + depth_mask = self.depth_mask + if depth_mask is not None: + clamp_bbox_xyxy_depth = rescale_bbox( + clamp_bbox_xyxy, tuple(depth_mask.shape[-2:]), effective_image_size_hw + ).long() + self.depth_mask = crop_around_box( + depth_mask, + clamp_bbox_xyxy_depth, + self.mask_path, # pyre-ignore + ) + + # changing principal_point according to bbox_crop + if self.camera is not None: + adjust_camera_to_bbox_crop_( + camera=self.camera, + image_size_wh=effective_image_size_hw.flip(dims=[-1]), + clamp_bbox_xywh=crop_bbox_xywh, + ) + + # pyre-ignore + self.effective_image_size_hw = crop_bbox_xywh[..., 2:].flip(dims=[-1]) + self._uncropped = False + + def resize_frame_(self, new_size_hw: torch.LongTensor) -> None: + """Resizes frame data in-place according to given dimensions. + + Args: + new_size_hw: target image size [height, width], a LongTensor of shape (2,) + + Raises: + ValueError: If the frame does not have an image size (usually a corner case + when no image has been loaded) + """ + + effective_image_size_hw = self.effective_image_size_hw + if effective_image_size_hw is None: + raise ValueError("Calling resize on image-less FrameData") + + image_height, image_width = new_size_hw.tolist() + + if self.fg_probability is not None: + self.fg_probability, _, _ = resize_image( + self.fg_probability, + image_height=image_height, + image_width=image_width, + mode="nearest", + ) + + if self.image_rgb is not None: + self.image_rgb, _, self.mask_crop = resize_image( + self.image_rgb, image_height=image_height, image_width=image_width + ) + + if self.depth_map is not None: + self.depth_map, _, _ = resize_image( + self.depth_map, + image_height=image_height, + image_width=image_width, + mode="nearest", + ) + + if self.depth_mask is not None: + self.depth_mask, _, _ = resize_image( + self.depth_mask, + image_height=image_height, + image_width=image_width, + mode="nearest", + ) + + if self.camera is not None: + if self.image_size_hw is None: + raise ValueError( + "image_size_hw has to be defined for resizing FrameData with cameras." + ) + adjust_camera_to_image_scale_( + camera=self.camera, + original_size_wh=effective_image_size_hw.flip(dims=[-1]), + new_size_wh=new_size_hw.flip(dims=[-1]), # pyre-ignore + ) + + self.effective_image_size_hw = new_size_hw + self._uncropped = False + + @classmethod + def collate(cls, batch): + """ + Given a list objects `batch` of class `cls`, collates them into a batched + representation suitable for processing with deep networks. + """ + + elem = batch[0] + + if isinstance(elem, cls): + pointcloud_ids = [id(el.sequence_point_cloud) for el in batch] + id_to_idx = defaultdict(list) + for i, pc_id in enumerate(pointcloud_ids): + id_to_idx[pc_id].append(i) + + sequence_point_cloud = [] + sequence_point_cloud_idx = -np.ones((len(batch),)) + for i, ind in enumerate(id_to_idx.values()): + sequence_point_cloud_idx[ind] = i + sequence_point_cloud.append(batch[ind[0]].sequence_point_cloud) + assert (sequence_point_cloud_idx >= 0).all() + + override_fields = { + "sequence_point_cloud": sequence_point_cloud, + "sequence_point_cloud_idx": sequence_point_cloud_idx.tolist(), + } + # note that the pre-collate value of sequence_point_cloud_idx is unused + + collated = {} + for f in fields(elem): + if not f.init: + continue + + list_values = override_fields.get( + f.name, [getattr(d, f.name) for d in batch] + ) + collated[f.name] = ( + cls.collate(list_values) + if all(list_value is not None for list_value in list_values) + else None + ) + return cls(**collated) + + elif isinstance(elem, Pointclouds): + return join_pointclouds_as_batch(batch) + + elif isinstance(elem, CamerasBase): + # TODO: don't store K; enforce working in NDC space + return join_cameras_as_batch(batch) + else: + return torch.utils.data.dataloader.default_collate(batch) + + +FrameDataSubtype = TypeVar("FrameDataSubtype", bound=FrameData) + + +class FrameDataBuilderBase(ReplaceableBase, Generic[FrameDataSubtype], ABC): + """A base class for FrameDataBuilders that build a FrameData object, load and + process the binary data (crop and resize). Implementations should parametrize + the class with a subtype of FrameData and set frame_data_type class variable to + that type. They have to also implement `build` method. + """ + + # To be initialised to FrameDataSubtype + frame_data_type: ClassVar[Type[FrameDataSubtype]] + + @abstractmethod + def build( + self, + frame_annotation: types.FrameAnnotation, + sequence_annotation: types.SequenceAnnotation, + *, + load_blobs: bool = True, + **kwargs, + ) -> FrameDataSubtype: + """An abstract method to build the frame data based on raw frame/sequence + annotations, load the binary data and adjust them according to the metadata. + """ + raise NotImplementedError() + + +class GenericFrameDataBuilder(FrameDataBuilderBase[FrameDataSubtype], ABC): + """ + A class to build a FrameData object, load and process the binary data (crop and + resize). This is an abstract class for extending to build FrameData subtypes. Most + users need to use concrete `FrameDataBuilder` class instead. + Beware that modifications of frame data are done in-place. + + Args: + dataset_root: The root folder of the dataset; all paths in frame / sequence + annotations are defined w.r.t. this root. Has to be set if any of the + load_* flabs below is true. + load_images: Enable loading the frame RGB data. + load_depths: Enable loading the frame depth maps. + load_depth_masks: Enable loading the frame depth map masks denoting the + depth values used for evaluation (the points consistent across views). + load_masks: Enable loading frame foreground masks. + load_point_clouds: Enable loading sequence-level point clouds. + max_points: Cap on the number of loaded points in the point cloud; + if reached, they are randomly sampled without replacement. + mask_images: Whether to mask the images with the loaded foreground masks; + 0 value is used for background. + mask_depths: Whether to mask the depth maps with the loaded foreground + masks; 0 value is used for background. + image_height: The height of the returned images, masks, and depth maps; + aspect ratio is preserved during cropping/resizing. + image_width: The width of the returned images, masks, and depth maps; + aspect ratio is preserved during cropping/resizing. + box_crop: Enable cropping of the image around the bounding box inferred + from the foreground region of the loaded segmentation mask; masks + and depth maps are cropped accordingly; cameras are corrected. + box_crop_mask_thr: The threshold used to separate pixels into foreground + and background based on the foreground_probability mask; if no value + is greater than this threshold, the loader lowers it and repeats. + box_crop_context: The amount of additional padding added to each + dimension of the cropping bounding box, relative to box size. + path_manager: Optionally a PathManager for interpreting paths in a special way. + """ + + dataset_root: Optional[str] = None + load_images: bool = True + load_depths: bool = True + load_depth_masks: bool = True + load_masks: bool = True + load_point_clouds: bool = False + max_points: int = 0 + mask_images: bool = False + mask_depths: bool = False + image_height: Optional[int] = 800 + image_width: Optional[int] = 800 + box_crop: bool = True + box_crop_mask_thr: float = 0.4 + box_crop_context: float = 0.3 + path_manager: Any = None + + def __post_init__(self) -> None: + load_any_blob = ( + self.load_images + or self.load_depths + or self.load_depth_masks + or self.load_masks + or self.load_point_clouds + ) + if load_any_blob and self.dataset_root is None: + raise ValueError( + "dataset_root must be set to load any blob data. " + "Make sure it is set in either FrameDataBuilder or Dataset params." + ) + + if load_any_blob and not self._exists_in_dataset_root(""): + raise ValueError( + f"dataset_root is passed but {self.dataset_root} does not exist." + ) + + def build( + self, + frame_annotation: types.FrameAnnotation, + sequence_annotation: types.SequenceAnnotation, + *, + load_blobs: bool = True, + **kwargs, + ) -> FrameDataSubtype: + """Builds the frame data based on raw frame/sequence annotations, loads the + binary data and adjust them according to the metadata. The processing includes: + * if box_crop is set, the image/mask/depth are cropped with the bounding + box provided or estimated from MaskAnnotation, + * if image_height/image_width are set, the image/mask/depth are resized to + fit that resolution. Note that the aspect ratio is preserved, and the + (possibly cropped) image is pasted into the top-left corner. In the + resulting frame_data, mask_crop field corresponds to the mask of the + pasted image. + + Args: + frame_annotation: frame annotation + sequence_annotation: sequence annotation + load_blobs: if the function should attempt loading the image, depth map + and mask, and foreground mask + + Returns: + The constructed FrameData object. + """ + + point_cloud = sequence_annotation.point_cloud + + frame_data = self.frame_data_type( + frame_number=safe_as_tensor(frame_annotation.frame_number, torch.long), + frame_timestamp=safe_as_tensor( + frame_annotation.frame_timestamp, torch.float + ), + sequence_name=frame_annotation.sequence_name, + sequence_category=sequence_annotation.category, + camera_quality_score=safe_as_tensor( + sequence_annotation.viewpoint_quality_score, torch.float + ), + point_cloud_quality_score=( + safe_as_tensor(point_cloud.quality_score, torch.float) + if point_cloud is not None + else None + ), + ) + + fg_mask_np: Optional[np.ndarray] = None + mask_annotation = frame_annotation.mask + if mask_annotation is not None: + if load_blobs and self.load_masks: + fg_mask_np, mask_path = self._load_fg_probability(frame_annotation) + frame_data.mask_path = mask_path + frame_data.fg_probability = safe_as_tensor(fg_mask_np, torch.float) + + bbox_xywh = mask_annotation.bounding_box_xywh + if bbox_xywh is None and fg_mask_np is not None: + bbox_xywh = get_bbox_from_mask(fg_mask_np, self.box_crop_mask_thr) + + frame_data.bbox_xywh = safe_as_tensor(bbox_xywh, torch.float) + + if frame_annotation.image is not None: + image_size_hw = safe_as_tensor(frame_annotation.image.size, torch.long) + frame_data.image_size_hw = image_size_hw # original image size + # image size after crop/resize + frame_data.effective_image_size_hw = image_size_hw + image_path = None + dataset_root = self.dataset_root + if frame_annotation.image.path is not None and dataset_root is not None: + image_path = os.path.join(dataset_root, frame_annotation.image.path) + frame_data.image_path = image_path + + if load_blobs and self.load_images: + if image_path is None: + raise ValueError("Image path is required to load images.") + + image_np = load_image(self._local_path(image_path)) + frame_data.image_rgb = self._postprocess_image( + image_np, frame_annotation.image.size, frame_data.fg_probability + ) + + if ( + load_blobs + and self.load_depths + and frame_annotation.depth is not None + and frame_annotation.depth.path is not None + ): + ( + frame_data.depth_map, + frame_data.depth_path, + frame_data.depth_mask, + ) = self._load_mask_depth(frame_annotation, fg_mask_np) + + if load_blobs and self.load_point_clouds and point_cloud is not None: + pcl_path = self._fix_point_cloud_path(point_cloud.path) + frame_data.sequence_point_cloud = load_pointcloud( + self._local_path(pcl_path), max_points=self.max_points + ) + frame_data.sequence_point_cloud_path = pcl_path + + if frame_annotation.viewpoint is not None: + frame_data.camera = self._get_pytorch3d_camera(frame_annotation) + + if self.box_crop: + frame_data.crop_by_metadata_bbox_(self.box_crop_context) + + if self.image_height is not None and self.image_width is not None: + new_size = (self.image_height, self.image_width) + frame_data.resize_frame_( + new_size_hw=torch.tensor(new_size, dtype=torch.long), # pyre-ignore + ) + + return frame_data + + def _load_fg_probability( + self, entry: types.FrameAnnotation + ) -> Tuple[np.ndarray, str]: + assert self.dataset_root is not None and entry.mask is not None + full_path = os.path.join(self.dataset_root, entry.mask.path) + fg_probability = load_mask(self._local_path(full_path)) + if fg_probability.shape[-2:] != entry.image.size: + raise ValueError( + f"bad mask size: {fg_probability.shape[-2:]} vs {entry.image.size}!" + ) + + return fg_probability, full_path + + def _postprocess_image( + self, + image_np: np.ndarray, + image_size: Tuple[int, int], + fg_probability: Optional[torch.Tensor], + ) -> torch.Tensor: + image_rgb = safe_as_tensor(image_np, torch.float) + + if image_rgb.shape[-2:] != image_size: + raise ValueError(f"bad image size: {image_rgb.shape[-2:]} vs {image_size}!") + + if self.mask_images: + assert fg_probability is not None + image_rgb *= fg_probability + + return image_rgb + + def _load_mask_depth( + self, + entry: types.FrameAnnotation, + fg_mask: Optional[np.ndarray], + ) -> Tuple[torch.Tensor, str, torch.Tensor]: + entry_depth = entry.depth + dataset_root = self.dataset_root + assert dataset_root is not None + assert entry_depth is not None and entry_depth.path is not None + path = os.path.join(dataset_root, entry_depth.path) + depth_map = load_depth(self._local_path(path), entry_depth.scale_adjustment) + + if self.mask_depths: + assert fg_mask is not None + depth_map *= fg_mask + + mask_path = entry_depth.mask_path + if self.load_depth_masks and mask_path is not None: + mask_path = os.path.join(dataset_root, mask_path) + depth_mask = load_depth_mask(self._local_path(mask_path)) + else: + depth_mask = (depth_map > 0.0).astype(np.float32) + + return torch.tensor(depth_map), path, torch.tensor(depth_mask) + + def _get_pytorch3d_camera( + self, + entry: types.FrameAnnotation, + ) -> PerspectiveCameras: + entry_viewpoint = entry.viewpoint + assert entry_viewpoint is not None + # principal point and focal length + principal_point = torch.tensor( + entry_viewpoint.principal_point, dtype=torch.float + ) + focal_length = torch.tensor(entry_viewpoint.focal_length, dtype=torch.float) + + format = entry_viewpoint.intrinsics_format + if entry_viewpoint.intrinsics_format == "ndc_norm_image_bounds": + # legacy PyTorch3D NDC format + # convert to pixels unequally and convert to ndc equally + image_size_as_list = list(reversed(entry.image.size)) + image_size_wh = torch.tensor(image_size_as_list, dtype=torch.float) + per_axis_scale = image_size_wh / image_size_wh.min() + focal_length = focal_length * per_axis_scale + principal_point = principal_point * per_axis_scale + elif entry_viewpoint.intrinsics_format != "ndc_isotropic": + raise ValueError(f"Unknown intrinsics format: {format}") + + return PerspectiveCameras( + focal_length=focal_length[None], + principal_point=principal_point[None], + R=torch.tensor(entry_viewpoint.R, dtype=torch.float)[None], + T=torch.tensor(entry_viewpoint.T, dtype=torch.float)[None], + ) + + def _fix_point_cloud_path(self, path: str) -> str: + """ + Fix up a point cloud path from the dataset. + Some files in Co3Dv2 have an accidental absolute path stored. + """ + unwanted_prefix = ( + "/large_experiments/p3/replay/datasets/co3d/co3d45k_220512/export_v23/" + ) + if path.startswith(unwanted_prefix): + path = path[len(unwanted_prefix) :] + assert self.dataset_root is not None + return os.path.join(self.dataset_root, path) + + def _local_path(self, path: str) -> str: + if self.path_manager is None: + return path + return self.path_manager.get_local_path(path) + + def _exists_in_dataset_root(self, relpath) -> bool: + if not self.dataset_root: + return False + + full_path = os.path.join(self.dataset_root, relpath) + if self.path_manager is None: + return os.path.exists(full_path) + else: + return self.path_manager.exists(full_path) + + +@registry.register +class FrameDataBuilder(GenericWorkaround, GenericFrameDataBuilder[FrameData]): + """ + A concrete class to build a FrameData object, load and process the binary data (crop + and resize). Beware that modifications of frame data are done in-place. Please see + the documentation for `GenericFrameDataBuilder` for the description of parameters + and methods. + """ + + frame_data_type: ClassVar[Type[FrameData]] = FrameData diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/json_index_dataset.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/json_index_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..69a7859aa77b887997555fd888a4411ff0937d56 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/json_index_dataset.py @@ -0,0 +1,671 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import copy +import functools +import gzip +import hashlib +import json +import logging +import os +import random +import warnings +from collections import defaultdict +from itertools import islice +from typing import ( + Any, + ClassVar, + Dict, + Iterable, + List, + Optional, + Sequence, + Tuple, + Type, + TYPE_CHECKING, + Union, +) + +from pytorch3d.implicitron.dataset import types +from pytorch3d.implicitron.dataset.dataset_base import DatasetBase +from pytorch3d.implicitron.dataset.frame_data import FrameData, FrameDataBuilder +from pytorch3d.implicitron.dataset.utils import is_known_frame_scalar +from pytorch3d.implicitron.tools.config import registry, ReplaceableBase +from pytorch3d.renderer.camera_utils import join_cameras_as_batch +from pytorch3d.renderer.cameras import CamerasBase + +from tqdm import tqdm + + +logger = logging.getLogger(__name__) + + +if TYPE_CHECKING: + from typing import TypedDict + + class FrameAnnotsEntry(TypedDict): + subset: Optional[str] + frame_annotation: types.FrameAnnotation + +else: + FrameAnnotsEntry = dict + + +@registry.register +class JsonIndexDataset(DatasetBase, ReplaceableBase): + """ + A dataset with annotations in json files like the Common Objects in 3D + (CO3D) dataset. + + Metadata-related args:: + frame_annotations_file: A zipped json file containing metadata of the + frames in the dataset, serialized List[types.FrameAnnotation]. + sequence_annotations_file: A zipped json file containing metadata of the + sequences in the dataset, serialized List[types.SequenceAnnotation]. + subset_lists_file: A json file containing the lists of frames corresponding + corresponding to different subsets (e.g. train/val/test) of the dataset; + format: {subset: (sequence_name, frame_id, file_path)}. + subsets: Restrict frames/sequences only to the given list of subsets + as defined in subset_lists_file (see above). + limit_to: Limit the dataset to the first #limit_to frames (after other + filters have been applied). + limit_sequences_to: Limit the dataset to the first + #limit_sequences_to sequences (after other sequence filters have been + applied but before frame-based filters). + pick_sequence: A list of sequence names to restrict the dataset to. + exclude_sequence: A list of the names of the sequences to exclude. + limit_category_to: Restrict the dataset to the given list of categories. + remove_empty_masks: Removes the frames with no active foreground pixels + in the segmentation mask after thresholding (see box_crop_mask_thr). + n_frames_per_sequence: If > 0, randomly samples #n_frames_per_sequence + frames in each sequences uniformly without replacement if it has + more frames than that; applied before other frame-level filters. + seed: The seed of the random generator sampling #n_frames_per_sequence + random frames per sequence. + sort_frames: Enable frame annotations sorting to group frames from the + same sequences together and order them by timestamps + eval_batches: A list of batches that form the evaluation set; + list of batch-sized lists of indices corresponding to __getitem__ + of this class, thus it can be used directly as a batch sampler. + eval_batch_index: + ( Optional[List[List[Union[Tuple[str, int, str], Tuple[str, int]]]] ) + A list of batches of frames described as (sequence_name, frame_idx) + that can form the evaluation set, `eval_batches` will be set from this. + + Blob-loading parameters: + dataset_root: The root folder of the dataset; all the paths in jsons are + specified relative to this root (but not json paths themselves). + load_images: Enable loading the frame RGB data. + load_depths: Enable loading the frame depth maps. + load_depth_masks: Enable loading the frame depth map masks denoting the + depth values used for evaluation (the points consistent across views). + load_masks: Enable loading frame foreground masks. + load_point_clouds: Enable loading sequence-level point clouds. + max_points: Cap on the number of loaded points in the point cloud; + if reached, they are randomly sampled without replacement. + mask_images: Whether to mask the images with the loaded foreground masks; + 0 value is used for background. + mask_depths: Whether to mask the depth maps with the loaded foreground + masks; 0 value is used for background. + image_height: The height of the returned images, masks, and depth maps; + aspect ratio is preserved during cropping/resizing. + image_width: The width of the returned images, masks, and depth maps; + aspect ratio is preserved during cropping/resizing. + box_crop: Enable cropping of the image around the bounding box inferred + from the foreground region of the loaded segmentation mask; masks + and depth maps are cropped accordingly; cameras are corrected. + box_crop_mask_thr: The threshold used to separate pixels into foreground + and background based on the foreground_probability mask; if no value + is greater than this threshold, the loader lowers it and repeats. + box_crop_context: The amount of additional padding added to each + dimension of the cropping bounding box, relative to box size. + """ + + frame_annotations_type: ClassVar[Type[types.FrameAnnotation]] = ( + types.FrameAnnotation + ) + + path_manager: Any = None + frame_annotations_file: str = "" + sequence_annotations_file: str = "" + subset_lists_file: str = "" + subsets: Optional[List[str]] = None + limit_to: int = 0 + limit_sequences_to: int = 0 + pick_sequence: Tuple[str, ...] = () + exclude_sequence: Tuple[str, ...] = () + limit_category_to: Tuple[int, ...] = () + dataset_root: str = "" + load_images: bool = True + load_depths: bool = True + load_depth_masks: bool = True + load_masks: bool = True + load_point_clouds: bool = False + max_points: int = 0 + mask_images: bool = False + mask_depths: bool = False + image_height: Optional[int] = 800 + image_width: Optional[int] = 800 + box_crop: bool = True + box_crop_mask_thr: float = 0.4 + box_crop_context: float = 0.3 + remove_empty_masks: bool = True + n_frames_per_sequence: int = -1 + seed: int = 0 + sort_frames: bool = False + eval_batches: Any = None + eval_batch_index: Any = None + # initialised in __post_init__ + # commented because of OmegaConf (for tests to pass) + # _frame_data_builder: FrameDataBuilder = field(init=False) + # frame_annots: List[FrameAnnotsEntry] = field(init=False) + # seq_annots: Dict[str, types.SequenceAnnotation] = field(init=False) + # _seq_to_idx: Dict[str, List[int]] = field(init=False) + + def __post_init__(self) -> None: + self._load_frames() + self._load_sequences() + if self.sort_frames: + self._sort_frames() + self._load_subset_lists() + self._filter_db() # also computes sequence indices + self._extract_and_set_eval_batches() + + # pyre-ignore + self._frame_data_builder = FrameDataBuilder( + dataset_root=self.dataset_root, + load_images=self.load_images, + load_depths=self.load_depths, + load_depth_masks=self.load_depth_masks, + load_masks=self.load_masks, + load_point_clouds=self.load_point_clouds, + max_points=self.max_points, + mask_images=self.mask_images, + mask_depths=self.mask_depths, + image_height=self.image_height, + image_width=self.image_width, + box_crop=self.box_crop, + box_crop_mask_thr=self.box_crop_mask_thr, + box_crop_context=self.box_crop_context, + path_manager=self.path_manager, + ) + logger.info(str(self)) + + def _extract_and_set_eval_batches(self) -> None: + """ + Sets eval_batches based on input eval_batch_index. + """ + if self.eval_batch_index is not None: + if self.eval_batches is not None: + raise ValueError( + "Cannot define both eval_batch_index and eval_batches." + ) + self.eval_batches = self.seq_frame_index_to_dataset_index( + self.eval_batch_index + ) + + def join(self, other_datasets: Iterable[DatasetBase]) -> None: + """ + Join the dataset with other JsonIndexDataset objects. + + Args: + other_datasets: A list of JsonIndexDataset objects to be joined + into the current dataset. + """ + if not all(isinstance(d, JsonIndexDataset) for d in other_datasets): + raise ValueError("This function can only join a list of JsonIndexDataset") + # pyre-ignore[16] + self.frame_annots.extend([fa for d in other_datasets for fa in d.frame_annots]) + # pyre-ignore[16] + self.seq_annots.update( + # https://gist.github.com/treyhunner/f35292e676efa0be1728 + functools.reduce( + lambda a, b: {**a, **b}, + # pyre-ignore[16] + [d.seq_annots for d in other_datasets], + ) + ) + all_eval_batches = [ + self.eval_batches, + *[d.eval_batches for d in other_datasets], # pyre-ignore[16] + ] + if not ( + all(ba is None for ba in all_eval_batches) + or all(ba is not None for ba in all_eval_batches) + ): + raise ValueError( + "When joining datasets, either all joined datasets have to have their" + " eval_batches defined, or all should have their eval batches undefined." + ) + if self.eval_batches is not None: + self.eval_batches = sum(all_eval_batches, []) + self._invalidate_indexes(filter_seq_annots=True) + + def is_filtered(self) -> bool: + """ + Returns `True` in case the dataset has been filtered and thus some frame annotations + stored on the disk might be missing in the dataset object. + + Returns: + is_filtered: `True` if the dataset has been filtered, else `False`. + """ + return ( + self.remove_empty_masks + or self.limit_to > 0 + or self.limit_sequences_to > 0 + or len(self.pick_sequence) > 0 + or len(self.exclude_sequence) > 0 + or len(self.limit_category_to) > 0 + or self.n_frames_per_sequence > 0 + ) + + def seq_frame_index_to_dataset_index( + self, + seq_frame_index: List[List[Union[Tuple[str, int, str], Tuple[str, int]]]], + allow_missing_indices: bool = False, + remove_missing_indices: bool = False, + suppress_missing_index_warning: bool = True, + ) -> Union[List[List[Optional[int]]], List[List[int]]]: + """ + Obtain indices into the dataset object given a list of frame ids. + + Args: + seq_frame_index: The list of frame ids specified as + `List[List[Tuple[sequence_name:str, frame_number:int]]]`. Optionally, + Image paths relative to the dataset_root can be stored specified as well: + `List[List[Tuple[sequence_name:str, frame_number:int, image_path:str]]]` + allow_missing_indices: If `False`, throws an IndexError upon reaching the first + entry from `seq_frame_index` which is missing in the dataset. + Otherwise, depending on `remove_missing_indices`, either returns `None` + in place of missing entries or removes the indices of missing entries. + remove_missing_indices: Active when `allow_missing_indices=True`. + If `False`, returns `None` in place of `seq_frame_index` entries that + are not present in the dataset. + If `True` removes missing indices from the returned indices. + suppress_missing_index_warning: + Active if `allow_missing_indices==True`. Suppressess a warning message + in case an entry from `seq_frame_index` is missing in the dataset + (expected in certain cases - e.g. when setting + `self.remove_empty_masks=True`). + + Returns: + dataset_idx: Indices of dataset entries corresponding to`seq_frame_index`. + """ + _dataset_seq_frame_n_index = { + seq: { + # pyre-ignore[16] + self.frame_annots[idx]["frame_annotation"].frame_number: idx + for idx in seq_idx + } + # pyre-ignore[16] + for seq, seq_idx in self._seq_to_idx.items() + } + + def _get_dataset_idx( + seq_name: str, frame_no: int, path: Optional[str] = None + ) -> Optional[int]: + idx_seq = _dataset_seq_frame_n_index.get(seq_name, None) + idx = idx_seq.get(frame_no, None) if idx_seq is not None else None + if idx is None: + msg = ( + f"sequence_name={seq_name} / frame_number={frame_no}" + " not in the dataset!" + ) + if not allow_missing_indices: + raise IndexError(msg) + if not suppress_missing_index_warning: + warnings.warn(msg) + return idx + if path is not None: + # Check that the loaded frame path is consistent + # with the one stored in self.frame_annots. + assert os.path.normpath( + # pyre-ignore[16] + self.frame_annots[idx]["frame_annotation"].image.path + ) == os.path.normpath( + path + ), f"Inconsistent frame indices {seq_name, frame_no, path}." + return idx + + dataset_idx = [ + [_get_dataset_idx(*b) for b in batch] # pyre-ignore [6] + for batch in seq_frame_index + ] + + if allow_missing_indices and remove_missing_indices: + # remove all None indices, and also batches with only None entries + valid_dataset_idx = [ + [b for b in batch if b is not None] for batch in dataset_idx + ] + return [batch for batch in valid_dataset_idx if len(batch) > 0] + + return dataset_idx + + def subset_from_frame_index( + self, + frame_index: List[Union[Tuple[str, int], Tuple[str, int, str]]], + allow_missing_indices: bool = True, + ) -> "JsonIndexDataset": + """ + Generate a dataset subset given the list of frames specified in `frame_index`. + + Args: + frame_index: The list of frame indentifiers (as stored in the metadata) + specified as `List[Tuple[sequence_name:str, frame_number:int]]`. Optionally, + Image paths relative to the dataset_root can be stored specified as well: + `List[Tuple[sequence_name:str, frame_number:int, image_path:str]]`, + in the latter case, if imaga_path do not match the stored paths, an error + is raised. + allow_missing_indices: If `False`, throws an IndexError upon reaching the first + entry from `frame_index` which is missing in the dataset. + Otherwise, generates a subset consisting of frames entries that actually + exist in the dataset. + """ + # Get the indices into the frame annots. + dataset_indices = self.seq_frame_index_to_dataset_index( + [frame_index], + allow_missing_indices=self.is_filtered() and allow_missing_indices, + )[0] + valid_dataset_indices = [i for i in dataset_indices if i is not None] + + # Deep copy the whole dataset except frame_annots, which are large so we + # deep copy only the requested subset of frame_annots. + memo = {id(self.frame_annots): None} # pyre-ignore[16] + dataset_new = copy.deepcopy(self, memo) + dataset_new.frame_annots = copy.deepcopy( + [self.frame_annots[i] for i in valid_dataset_indices] + ) + + # This will kill all unneeded sequence annotations. + dataset_new._invalidate_indexes(filter_seq_annots=True) + + # Finally annotate the frame annotations with the name of the subset + # stored in meta. + for frame_annot in dataset_new.frame_annots: + frame_annotation = frame_annot["frame_annotation"] + if frame_annotation.meta is not None: + frame_annot["subset"] = frame_annotation.meta.get("frame_type", None) + + # A sanity check - this will crash in case some entries from frame_index are missing + # in dataset_new. + valid_frame_index = [ + fi for fi, di in zip(frame_index, dataset_indices) if di is not None + ] + dataset_new.seq_frame_index_to_dataset_index( + [valid_frame_index], allow_missing_indices=False + ) + + return dataset_new + + def __str__(self) -> str: + # pyre-ignore[16] + return f"JsonIndexDataset #frames={len(self.frame_annots)}" + + def __len__(self) -> int: + # pyre-ignore[16] + return len(self.frame_annots) + + def _get_frame_type(self, entry: FrameAnnotsEntry) -> Optional[str]: + return entry["subset"] + + def get_all_train_cameras(self) -> CamerasBase: + """ + Returns the cameras corresponding to all the known frames. + """ + logger.info("Loading all train cameras.") + cameras = [] + # pyre-ignore[16] + for frame_idx, frame_annot in enumerate(tqdm(self.frame_annots)): + frame_type = self._get_frame_type(frame_annot) + if frame_type is None: + raise ValueError("subsets not loaded") + if is_known_frame_scalar(frame_type): + cameras.append(self[frame_idx].camera) + return join_cameras_as_batch(cameras) + + def __getitem__(self, index) -> FrameData: + # pyre-ignore[16] + if index >= len(self.frame_annots): + raise IndexError(f"index {index} out of range {len(self.frame_annots)}") + + entry = self.frame_annots[index]["frame_annotation"] + + # pyre-ignore + frame_data = self._frame_data_builder.build( + entry, + # pyre-ignore + self.seq_annots[entry.sequence_name], + ) + # Optional field + frame_data.frame_type = self._get_frame_type(self.frame_annots[index]) + + return frame_data + + def _load_frames(self) -> None: + logger.info(f"Loading Co3D frames from {self.frame_annotations_file}.") + local_file = self._local_path(self.frame_annotations_file) + with gzip.open(local_file, "rt", encoding="utf8") as zipfile: + frame_annots_list = types.load_dataclass( + zipfile, List[self.frame_annotations_type] + ) + if not frame_annots_list: + raise ValueError("Empty dataset!") + # pyre-ignore[16] + self.frame_annots = [ + FrameAnnotsEntry(frame_annotation=a, subset=None) for a in frame_annots_list + ] + + def _load_sequences(self) -> None: + logger.info(f"Loading Co3D sequences from {self.sequence_annotations_file}.") + local_file = self._local_path(self.sequence_annotations_file) + with gzip.open(local_file, "rt", encoding="utf8") as zipfile: + seq_annots = types.load_dataclass(zipfile, List[types.SequenceAnnotation]) + if not seq_annots: + raise ValueError("Empty sequences file!") + # pyre-ignore[16] + self.seq_annots = {entry.sequence_name: entry for entry in seq_annots} + + def _load_subset_lists(self) -> None: + logger.info(f"Loading Co3D subset lists from {self.subset_lists_file}.") + if not self.subset_lists_file: + return + + with open(self._local_path(self.subset_lists_file), "r") as f: + subset_to_seq_frame = json.load(f) + + frame_path_to_subset = { + path: subset + for subset, frames in subset_to_seq_frame.items() + for _, _, path in frames + } + # pyre-ignore[16] + for frame in self.frame_annots: + frame["subset"] = frame_path_to_subset.get( + frame["frame_annotation"].image.path, None + ) + if frame["subset"] is None: + warnings.warn( + "Subset lists are given but don't include " + + frame["frame_annotation"].image.path + ) + + def _sort_frames(self) -> None: + # Sort frames to have them grouped by sequence, ordered by timestamp + # pyre-ignore[16] + self.frame_annots = sorted( + self.frame_annots, + key=lambda f: ( + f["frame_annotation"].sequence_name, + f["frame_annotation"].frame_timestamp or 0, + ), + ) + + def _filter_db(self) -> None: + if self.remove_empty_masks: + logger.info("Removing images with empty masks.") + # pyre-ignore[16] + old_len = len(self.frame_annots) + + msg = "remove_empty_masks needs every MaskAnnotation.mass to be set." + + def positive_mass(frame_annot: types.FrameAnnotation) -> bool: + mask = frame_annot.mask + if mask is None: + return False + if mask.mass is None: + raise ValueError(msg) + return mask.mass > 1 + + self.frame_annots = [ + frame + for frame in self.frame_annots + if positive_mass(frame["frame_annotation"]) + ] + logger.info("... filtered %d -> %d" % (old_len, len(self.frame_annots))) + + # this has to be called after joining with categories!! + subsets = self.subsets + if subsets: + if not self.subset_lists_file: + raise ValueError( + "Subset filter is on but subset_lists_file was not given" + ) + + logger.info(f"Limiting Co3D dataset to the '{subsets}' subsets.") + + # truncate the list of subsets to the valid one + self.frame_annots = [ + entry for entry in self.frame_annots if entry["subset"] in subsets + ] + if len(self.frame_annots) == 0: + raise ValueError(f"There are no frames in the '{subsets}' subsets!") + + self._invalidate_indexes(filter_seq_annots=True) + + if len(self.limit_category_to) > 0: + logger.info(f"Limiting dataset to categories: {self.limit_category_to}") + # pyre-ignore[16] + self.seq_annots = { + name: entry + for name, entry in self.seq_annots.items() + if entry.category in self.limit_category_to + } + + # sequence filters + for prefix in ("pick", "exclude"): + orig_len = len(self.seq_annots) + attr = f"{prefix}_sequence" + arr = getattr(self, attr) + if len(arr) > 0: + logger.info(f"{attr}: {str(arr)}") + self.seq_annots = { + name: entry + for name, entry in self.seq_annots.items() + if (name in arr) == (prefix == "pick") + } + logger.info("... filtered %d -> %d" % (orig_len, len(self.seq_annots))) + + if self.limit_sequences_to > 0: + self.seq_annots = dict( + islice(self.seq_annots.items(), self.limit_sequences_to) + ) + + # retain only frames from retained sequences + self.frame_annots = [ + f + for f in self.frame_annots + if f["frame_annotation"].sequence_name in self.seq_annots + ] + + self._invalidate_indexes() + + if self.n_frames_per_sequence > 0: + logger.info(f"Taking max {self.n_frames_per_sequence} per sequence.") + keep_idx = [] + # pyre-ignore[16] + for seq, seq_indices in self._seq_to_idx.items(): + # infer the seed from the sequence name, this is reproducible + # and makes the selection differ for different sequences + seed = _seq_name_to_seed(seq) + self.seed + seq_idx_shuffled = random.Random(seed).sample( + sorted(seq_indices), len(seq_indices) + ) + keep_idx.extend(seq_idx_shuffled[: self.n_frames_per_sequence]) + + logger.info( + "... filtered %d -> %d" % (len(self.frame_annots), len(keep_idx)) + ) + self.frame_annots = [self.frame_annots[i] for i in keep_idx] + self._invalidate_indexes(filter_seq_annots=False) + # sequences are not decimated, so self.seq_annots is valid + + if self.limit_to > 0 and self.limit_to < len(self.frame_annots): + logger.info( + "limit_to: filtered %d -> %d" % (len(self.frame_annots), self.limit_to) + ) + self.frame_annots = self.frame_annots[: self.limit_to] + self._invalidate_indexes(filter_seq_annots=True) + + def _invalidate_indexes(self, filter_seq_annots: bool = False) -> None: + # update _seq_to_idx and filter seq_meta according to frame_annots change + # if filter_seq_annots, also uldates seq_annots based on the changed _seq_to_idx + self._invalidate_seq_to_idx() + + if filter_seq_annots: + # pyre-ignore[16] + self.seq_annots = { + k: v + for k, v in self.seq_annots.items() + # pyre-ignore[16] + if k in self._seq_to_idx + } + + def _invalidate_seq_to_idx(self) -> None: + seq_to_idx = defaultdict(list) + # pyre-ignore[16] + for idx, entry in enumerate(self.frame_annots): + seq_to_idx[entry["frame_annotation"].sequence_name].append(idx) + # pyre-ignore[16] + self._seq_to_idx = seq_to_idx + + def _local_path(self, path: str) -> str: + if self.path_manager is None: + return path + return self.path_manager.get_local_path(path) + + def get_frame_numbers_and_timestamps( + self, idxs: Sequence[int], subset_filter: Optional[Sequence[str]] = None + ) -> List[Tuple[int, float]]: + out: List[Tuple[int, float]] = [] + for idx in idxs: + if ( + subset_filter is not None + # pyre-fixme[16]: `JsonIndexDataset` has no attribute `frame_annots`. + and self.frame_annots[idx]["subset"] not in subset_filter + ): + continue + + frame_annotation = self.frame_annots[idx]["frame_annotation"] + out.append( + (frame_annotation.frame_number, frame_annotation.frame_timestamp) + ) + return out + + def category_to_sequence_names(self) -> Dict[str, List[str]]: + c2seq = defaultdict(list) + # pyre-ignore + for sequence_name, sa in self.seq_annots.items(): + c2seq[sa.category].append(sequence_name) + return dict(c2seq) + + def get_eval_batches(self) -> Optional[List[List[int]]]: + return self.eval_batches + + +def _seq_name_to_seed(seq_name) -> int: + return int(hashlib.sha1(seq_name.encode("utf-8")).hexdigest(), 16) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/json_index_dataset_map_provider.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/json_index_dataset_map_provider.py new file mode 100644 index 0000000000000000000000000000000000000000..106a2e4c000664a913d9974b687bcf93dea35386 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/json_index_dataset_map_provider.py @@ -0,0 +1,320 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +import json +import os +from typing import Dict, List, Optional, Tuple, Type + +from omegaconf import DictConfig +from pytorch3d.implicitron.tools.config import ( + expand_args_fields, + registry, + run_auto_creation, +) +from pytorch3d.renderer.cameras import CamerasBase + +from .dataset_map_provider import DatasetMap, DatasetMapProviderBase, PathManagerFactory +from .json_index_dataset import JsonIndexDataset + +from .utils import ( + DATASET_TYPE_KNOWN, + DATASET_TYPE_TEST, + DATASET_TYPE_TRAIN, + DATASET_TYPE_UNKNOWN, +) + + +# fmt: off +CO3D_CATEGORIES: List[str] = list(reversed([ + "baseballbat", "banana", "bicycle", "microwave", "tv", + "cellphone", "toilet", "hairdryer", "couch", "kite", "pizza", + "umbrella", "wineglass", "laptop", + "hotdog", "stopsign", "frisbee", "baseballglove", + "cup", "parkingmeter", "backpack", "toyplane", "toybus", + "handbag", "chair", "keyboard", "car", "motorcycle", + "carrot", "bottle", "sandwich", "remote", "bowl", "skateboard", + "toaster", "mouse", "toytrain", "book", "toytruck", + "orange", "broccoli", "plant", "teddybear", + "suitcase", "bench", "ball", "cake", + "vase", "hydrant", "apple", "donut", +])) +# fmt: on + +_CO3D_DATASET_ROOT: str = os.getenv("CO3D_DATASET_ROOT", "") + +# _NEED_CONTROL is a list of those elements of JsonIndexDataset which +# are not directly specified for it in the config but come from the +# DatasetMapProvider. +_NEED_CONTROL: Tuple[str, ...] = ( + "dataset_root", + "eval_batches", + "eval_batch_index", + "n_frames_per_sequence", + "path_manager", + "pick_sequence", + "subsets", + "frame_annotations_file", + "sequence_annotations_file", + "subset_lists_file", +) + + +@registry.register +class JsonIndexDatasetMapProvider(DatasetMapProviderBase): # pyre-ignore [13] + """ + Generates the training / validation and testing dataset objects for + a dataset laid out on disk like Co3D, with annotations in json files. + + Args: + category: The object category of the dataset. + task_str: "multisequence" or "singlesequence". + dataset_root: The root folder of the dataset. + n_frames_per_sequence: Randomly sample #n_frames_per_sequence frames + in each sequence. + test_on_train: Construct validation and test datasets from + the training subset. + restrict_sequence_name: Restrict the dataset sequences to the ones + present in the given list of names. + test_restrict_sequence_id: The ID of the loaded sequence. + Active for task_str='singlesequence'. + assert_single_seq: Assert that only frames from a single sequence + are present in all generated datasets. + only_test_set: Load only the test set. + dataset_class_type: name of class (JsonIndexDataset or a subclass) + to use for the dataset. + dataset_X_args (e.g. dataset_JsonIndexDataset_args): arguments passed + to all the dataset constructors. + path_manager_factory: (Optional) An object that generates an instance of + PathManager that can translate provided file paths. + path_manager_factory_class_type: The class type of `path_manager_factory`. + """ + + category: str + task_str: str = "singlesequence" + dataset_root: str = _CO3D_DATASET_ROOT + n_frames_per_sequence: int = -1 + test_on_train: bool = False + restrict_sequence_name: Tuple[str, ...] = () + test_restrict_sequence_id: int = -1 + assert_single_seq: bool = False + only_test_set: bool = False + dataset: JsonIndexDataset + dataset_class_type: str = "JsonIndexDataset" + path_manager_factory: PathManagerFactory + path_manager_factory_class_type: str = "PathManagerFactory" + + @classmethod + def dataset_tweak_args(cls, type, args: DictConfig) -> None: + """ + Called by get_default_args(JsonIndexDatasetMapProvider) to + not expose certain fields of each dataset class. + """ + for key in _NEED_CONTROL: + del args[key] + + def create_dataset(self): + """ + Prevent the member named dataset from being created. + """ + return + + def __post_init__(self): + super().__init__() + run_auto_creation(self) + if self.only_test_set and self.test_on_train: + raise ValueError("Cannot have only_test_set and test_on_train") + + path_manager = self.path_manager_factory.get() + + # TODO: + # - implement loading multiple categories + + frame_file = os.path.join( + self.dataset_root, self.category, "frame_annotations.jgz" + ) + sequence_file = os.path.join( + self.dataset_root, self.category, "sequence_annotations.jgz" + ) + subset_lists_file = os.path.join( + self.dataset_root, self.category, "set_lists.json" + ) + common_kwargs = { + "dataset_root": self.dataset_root, + "path_manager": path_manager, + "frame_annotations_file": frame_file, + "sequence_annotations_file": sequence_file, + "subset_lists_file": subset_lists_file, + **getattr(self, f"dataset_{self.dataset_class_type}_args"), + } + + # This maps the common names of the dataset subsets ("train"/"val"/"test") + # to the names of the subsets in the CO3D dataset. + set_names_mapping = _get_co3d_set_names_mapping( + self.task_str, + self.test_on_train, + self.only_test_set, + ) + + # load the evaluation batches + batch_indices_path = os.path.join( + self.dataset_root, + self.category, + f"eval_batches_{self.task_str}.json", + ) + if path_manager is not None: + batch_indices_path = path_manager.get_local_path(batch_indices_path) + if not os.path.isfile(batch_indices_path): + # The batch indices file does not exist. + # Most probably the user has not specified the root folder. + raise ValueError( + f"Looking for batch indices in {batch_indices_path}. " + + "Please specify a correct dataset_root folder." + ) + + with open(batch_indices_path, "r") as f: + eval_batch_index = json.load(f) + restrict_sequence_name = self.restrict_sequence_name + + if self.task_str == "singlesequence": + if ( + self.test_restrict_sequence_id is None + or self.test_restrict_sequence_id < 0 + ): + raise ValueError( + "Please specify an integer id 'test_restrict_sequence_id'" + + " of the sequence considered for 'singlesequence'" + + " training and evaluation." + ) + if len(self.restrict_sequence_name) > 0: + raise ValueError( + "For the 'singlesequence' task, the restrict_sequence_name has" + " to be unset while test_restrict_sequence_id has to be set to an" + " integer defining the order of the evaluation sequence." + ) + # a sort-stable set() equivalent: + eval_batches_sequence_names = list( + {b[0][0]: None for b in eval_batch_index}.keys() + ) + eval_sequence_name = eval_batches_sequence_names[ + self.test_restrict_sequence_id + ] + eval_batch_index = [ + b for b in eval_batch_index if b[0][0] == eval_sequence_name + ] + # overwrite the restrict_sequence_name + restrict_sequence_name = [eval_sequence_name] + if len(restrict_sequence_name) > 0: + eval_batch_index = [ + b for b in eval_batch_index if b[0][0] in restrict_sequence_name + ] + + dataset_type: Type[JsonIndexDataset] = registry.get( + JsonIndexDataset, self.dataset_class_type + ) + expand_args_fields(dataset_type) + train_dataset = None + if not self.only_test_set: + train_dataset = dataset_type( + n_frames_per_sequence=self.n_frames_per_sequence, + subsets=set_names_mapping["train"], + pick_sequence=restrict_sequence_name, + **common_kwargs, + ) + if self.test_on_train: + assert train_dataset is not None + val_dataset = test_dataset = train_dataset + else: + val_dataset = dataset_type( + n_frames_per_sequence=-1, + subsets=set_names_mapping["val"], + pick_sequence=restrict_sequence_name, + **common_kwargs, + ) + test_dataset = dataset_type( + n_frames_per_sequence=-1, + subsets=set_names_mapping["test"], + pick_sequence=restrict_sequence_name, + eval_batch_index=eval_batch_index, + **common_kwargs, + ) + dataset_map = DatasetMap( + train=train_dataset, val=val_dataset, test=test_dataset + ) + + if self.assert_single_seq: + # check there's only one sequence in all datasets + sequence_names = { + sequence_name + for dset in dataset_map.iter_datasets() + for sequence_name in dset.sequence_names() + } + if len(sequence_names) > 1: + raise ValueError("Multiple sequences loaded but expected one") + + self.dataset_map = dataset_map + + def get_dataset_map(self) -> DatasetMap: + # pyre-ignore[16] + return self.dataset_map + + def get_all_train_cameras(self) -> Optional[CamerasBase]: + if self.task_str == "multisequence": + return None + + assert self.task_str == "singlesequence" + + # pyre-ignore[16] + train_dataset = self.dataset_map.train + assert isinstance(train_dataset, JsonIndexDataset) + return train_dataset.get_all_train_cameras() + + +def _get_co3d_set_names_mapping( + task_str: str, + test_on_train: bool, + only_test: bool, +) -> Dict[str, List[str]]: + """ + Returns the mapping of the common dataset subset names ("train"/"val"/"test") + to the names of the corresponding subsets in the CO3D dataset + ("test_known"/"test_unseen"/"train_known"/"train_unseen"). + + The keys returned will be + - train (if not only_test) + - val (if not test_on_train) + - test (if not test_on_train) + """ + single_seq = task_str == "singlesequence" + + if only_test: + set_names_mapping = {} + else: + set_names_mapping = { + "train": [ + (DATASET_TYPE_TEST if single_seq else DATASET_TYPE_TRAIN) + + "_" + + DATASET_TYPE_KNOWN + ] + } + if not test_on_train: + prefixes = [DATASET_TYPE_TEST] + if not single_seq: + prefixes.append(DATASET_TYPE_TRAIN) + set_names_mapping.update( + { + dset: [ + p + "_" + t + for p in prefixes + for t in [DATASET_TYPE_KNOWN, DATASET_TYPE_UNKNOWN] + ] + for dset in ["val", "test"] + } + ) + + return set_names_mapping diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/json_index_dataset_map_provider_v2.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/json_index_dataset_map_provider_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..f74f844caa12b1a0416358948b7c3b3b2b0c8a46 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/json_index_dataset_map_provider_v2.py @@ -0,0 +1,479 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +import copy +import json +import logging +import multiprocessing +import os +import warnings +from collections import defaultdict +from typing import Dict, List, Optional, Tuple, Type, Union + +import numpy as np +from iopath.common.file_io import PathManager + +from omegaconf import DictConfig +from pytorch3d.implicitron.dataset.dataset_map_provider import ( + DatasetMap, + DatasetMapProviderBase, + PathManagerFactory, +) +from pytorch3d.implicitron.dataset.json_index_dataset import JsonIndexDataset +from pytorch3d.implicitron.tools.config import ( + expand_args_fields, + registry, + run_auto_creation, +) + +from pytorch3d.renderer.cameras import CamerasBase +from tqdm import tqdm + + +_CO3DV2_DATASET_ROOT: str = os.getenv("CO3DV2_DATASET_ROOT", "") + +# _NEED_CONTROL is a list of those elements of JsonIndexDataset which +# are not directly specified for it in the config but come from the +# DatasetMapProvider. +_NEED_CONTROL: Tuple[str, ...] = ( + "dataset_root", + "eval_batches", + "eval_batch_index", + "path_manager", + "subsets", + "frame_annotations_file", + "sequence_annotations_file", + "subset_lists_file", +) + +logger = logging.getLogger(__name__) + + +@registry.register +class JsonIndexDatasetMapProviderV2(DatasetMapProviderBase): # pyre-ignore [13] + """ + Generates the training, validation, and testing dataset objects for + a dataset laid out on disk like CO3Dv2, with annotations in gzipped json files. + + The dataset is organized in the filesystem as follows:: + + self.dataset_root + ├── + │ ├── + │ │ ├── depth_masks + │ │ ├── depths + │ │ ├── images + │ │ ├── masks + │ │ └── pointcloud.ply + │ ├── + │ │ ├── depth_masks + │ │ ├── depths + │ │ ├── images + │ │ ├── masks + │ │ └── pointcloud.ply + │ ├── ... + │ ├── + │ ├── set_lists + │ ├── set_lists_.json + │ ├── set_lists_.json + │ ├── ... + │ ├── set_lists_.json + │ ├── eval_batches + │ │ ├── eval_batches_.json + │ │ ├── eval_batches_.json + │ │ ├── ... + │ │ ├── eval_batches_.json + │ ├── frame_annotations.jgz + │ ├── sequence_annotations.jgz + ├── + ├── ... + ├── + + The dataset contains sequences named `` from `K` categories with + names ``. Each category comprises sequence folders + `/` containing the list of sequence images, depth maps, + foreground masks, and valid-depth masks `images`, `depths`, `masks`, and `depth_masks` + respectively. Furthermore, `//set_lists/` stores `M` + json files `set_lists_.json`, each describing a certain sequence subset. + + Users specify the loaded dataset subset by setting `self.subset_name` to one of the + available subset names ``. + + `frame_annotations.jgz` and `sequence_annotations.jgz` are gzipped json files containing + the list of all frames and sequences of the given category stored as lists of + `FrameAnnotation` and `SequenceAnnotation` objects respectivelly. + + Each `set_lists_.json` file contains the following dictionary:: + + { + "train": [ + (sequence_name: str, frame_number: int, image_path: str), + ... + ], + "val": [ + (sequence_name: str, frame_number: int, image_path: str), + ... + ], + "test": [ + (sequence_name: str, frame_number: int, image_path: str), + ... + ], + ] + + defining the list of frames (identified with their `sequence_name` and `frame_number`) + in the "train", "val", and "test" subsets of the dataset. + Note that `frame_number` can be obtained only from `frame_annotations.jgz` and + does not necesarrily correspond to the numeric suffix of the corresponding image + file name (e.g. a file `//images/frame00005.jpg` can + have its frame number set to `20`, not 5). + + Each `eval_batches_.json` file contains a list of evaluation examples + in the following form:: + + [ + [ # batch 1 + (sequence_name: str, frame_number: int, image_path: str), + ... + ], + [ # batch 1 + (sequence_name: str, frame_number: int, image_path: str), + ... + ], + ] + + Note that the evaluation examples always come from the `"test"` subset of the dataset. + (test frames can repeat across batches). + + Args: + category: Dataset categories to load expressed as a string of comma-separated + category names (e.g. `"apple,car,orange"`). + subset_name: The name of the dataset subset. For CO3Dv2, these include + e.g. "manyview_dev_0", "fewview_test", ... + dataset_root: The root folder of the dataset. + test_on_train: Construct validation and test datasets from + the training subset. + only_test_set: Load only the test set. Incompatible with `test_on_train`. + load_eval_batches: Load the file containing eval batches pointing to the + test dataset. + n_known_frames_for_test: Add a certain number of known frames to each + eval batch. Useful for evaluating models that require + source views as input (e.g. NeRF-WCE / PixelNeRF). + dataset_args: Specifies additional arguments to the + JsonIndexDataset constructor call. + path_manager_factory: (Optional) An object that generates an instance of + PathManager that can translate provided file paths. + path_manager_factory_class_type: The class type of `path_manager_factory`. + """ + + category: str + subset_name: str + dataset_root: str = _CO3DV2_DATASET_ROOT + + test_on_train: bool = False + only_test_set: bool = False + load_eval_batches: bool = True + num_load_workers: int = 4 + + n_known_frames_for_test: int = 0 + + dataset_class_type: str = "JsonIndexDataset" + dataset: JsonIndexDataset + + path_manager_factory: PathManagerFactory + path_manager_factory_class_type: str = "PathManagerFactory" + + def __post_init__(self): + super().__init__() + run_auto_creation(self) + + if self.only_test_set and self.test_on_train: + raise ValueError("Cannot have only_test_set and test_on_train") + + if "," in self.category: + # a comma-separated list of categories to load + categories = [c.strip() for c in self.category.split(",")] + logger.info(f"Loading a list of categories: {str(categories)}.") + with multiprocessing.Pool( + processes=min(self.num_load_workers, len(categories)) + ) as pool: + category_dataset_maps = list( + tqdm( + pool.imap(self._load_category, categories), + total=len(categories), + ) + ) + dataset_map = category_dataset_maps[0] + dataset_map.join(category_dataset_maps[1:]) + + else: + # one category to load + dataset_map = self._load_category(self.category) + + self.dataset_map = dataset_map + + def _load_category(self, category: str) -> DatasetMap: + + frame_file = os.path.join(self.dataset_root, category, "frame_annotations.jgz") + sequence_file = os.path.join( + self.dataset_root, category, "sequence_annotations.jgz" + ) + + path_manager = self.path_manager_factory.get() + + if path_manager is not None: + path_managed_frame_file = path_manager.get_local_path(frame_file) + else: + path_managed_frame_file = frame_file + if not os.path.isfile(path_managed_frame_file): + # The frame_file does not exist. + # Most probably the user has not specified the root folder. + raise ValueError( + f"Looking for frame annotations in {path_managed_frame_file}." + + " Please specify a correct dataset_root folder." + + " Note: By default the root folder is taken from the" + + " CO3DV2_DATASET_ROOT environment variable." + ) + + # setup the common dataset arguments + common_dataset_kwargs = getattr(self, f"dataset_{self.dataset_class_type}_args") + common_dataset_kwargs = { + **common_dataset_kwargs, + "dataset_root": self.dataset_root, + "frame_annotations_file": frame_file, + "sequence_annotations_file": sequence_file, + "subsets": None, + "subset_lists_file": "", + "path_manager": path_manager, + } + + # get the used dataset type + dataset_type: Type[JsonIndexDataset] = registry.get( + JsonIndexDataset, self.dataset_class_type + ) + expand_args_fields(dataset_type) + + dataset = dataset_type(**common_dataset_kwargs) + + available_subset_names = self._get_available_subset_names(category) + logger.debug(f"Available subset names: {str(available_subset_names)}.") + if self.subset_name not in available_subset_names: + raise ValueError( + f"Unknown subset name {self.subset_name}." + + f" Choose one of available subsets: {str(available_subset_names)}." + ) + + # load the list of train/val/test frames + subset_mapping = self._load_annotation_json( + os.path.join(category, "set_lists", f"set_lists_{self.subset_name}.json") + ) + + # load the evaluation batches + if self.load_eval_batches: + eval_batch_index = self._load_annotation_json( + os.path.join( + category, + "eval_batches", + f"eval_batches_{self.subset_name}.json", + ) + ) + else: + eval_batch_index = None + + train_dataset = None + if not self.only_test_set: + # load the training set + logger.debug("Extracting train dataset.") + train_dataset = dataset.subset_from_frame_index(subset_mapping["train"]) + logger.info(f"Train dataset: {str(train_dataset)}") + + if self.test_on_train: + assert train_dataset is not None + val_dataset = test_dataset = train_dataset + else: + # load the val and test sets + logger.debug("Extracting val dataset.") + val_dataset = dataset.subset_from_frame_index(subset_mapping["val"]) + logger.info(f"Val dataset: {str(val_dataset)}") + logger.debug("Extracting test dataset.") + + if (self.n_known_frames_for_test > 0) and self.load_eval_batches: + # extend the test subset mapping and the dataset with additional + # known views from the train dataset + ( + eval_batch_index, + subset_mapping["test"], + ) = self._extend_test_data_with_known_views( + subset_mapping, + eval_batch_index, + ) + + test_dataset = dataset.subset_from_frame_index(subset_mapping["test"]) + logger.info(f"Test dataset: {str(test_dataset)}") + if self.load_eval_batches: + # load the eval batches + logger.debug("Extracting eval batches.") + try: + test_dataset.eval_batches = ( + test_dataset.seq_frame_index_to_dataset_index( + eval_batch_index, + ) + ) + except IndexError: + warnings.warn( + "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n" + + "Some eval batches are missing from the test dataset.\n" + + "The evaluation results will be incomparable to the\n" + + "evaluation results calculated on the original dataset.\n" + + "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@" + ) + test_dataset.eval_batches = ( + test_dataset.seq_frame_index_to_dataset_index( + eval_batch_index, + allow_missing_indices=True, + remove_missing_indices=True, + ) + ) + logger.info(f"# eval batches: {len(test_dataset.eval_batches)}") + + return DatasetMap(train=train_dataset, val=val_dataset, test=test_dataset) + + @classmethod + def dataset_tweak_args(cls, type, args: DictConfig) -> None: + """ + Called by get_default_args(JsonIndexDatasetMapProviderV2) to + not expose certain fields of each dataset class. + """ + for key in _NEED_CONTROL: + del args[key] + + def create_dataset(self): + # The dataset object is created inside `self.get_dataset_map` + pass + + def get_dataset_map(self) -> DatasetMap: + return self.dataset_map # pyre-ignore [16] + + def get_category_to_subset_name_list(self) -> Dict[str, List[str]]: + """ + Returns a global dataset index containing the available subset names per category + as a dictionary. + + Returns: + category_to_subset_name_list: A dictionary containing subset names available + per category of the following form:: + + { + category_0: [category_0_subset_name_0, category_0_subset_name_1, ...], + category_1: [category_1_subset_name_0, category_1_subset_name_1, ...], + ... + } + + """ + category_to_subset_name_list_json = "category_to_subset_name_list.json" + category_to_subset_name_list = self._load_annotation_json( + category_to_subset_name_list_json + ) + return category_to_subset_name_list + + def get_all_train_cameras(self) -> Optional[CamerasBase]: + # pyre-ignore[16] + train_dataset = self.dataset_map.train + assert isinstance(train_dataset, JsonIndexDataset) + return train_dataset.get_all_train_cameras() + + def _load_annotation_json(self, json_filename: str): + full_path = os.path.join( + self.dataset_root, + json_filename, + ) + logger.info(f"Loading frame index json from {full_path}.") + path_manager = self.path_manager_factory.get() + if path_manager is not None: + full_path = path_manager.get_local_path(full_path) + if not os.path.isfile(full_path): + # The batch indices file does not exist. + # Most probably the user has not specified the root folder. + raise ValueError( + f"Looking for dataset json file in {full_path}. " + + "Please specify a correct dataset_root folder." + ) + with open(full_path, "r") as f: + data = json.load(f) + return data + + def _get_available_subset_names(self, category: str): + return get_available_subset_names( + self.dataset_root, + category, + path_manager=self.path_manager_factory.get(), + ) + + def _extend_test_data_with_known_views( + self, + subset_mapping: Dict[str, List[Union[Tuple[str, int], Tuple[str, int, str]]]], + eval_batch_index: List[List[Union[Tuple[str, int, str], Tuple[str, int]]]], + ): + # convert the train subset mapping to a dict: + # sequence_to_train_frames: {sequence_name: frame_index} + sequence_to_train_frames = defaultdict(list) + for frame_entry in subset_mapping["train"]: + sequence_name = frame_entry[0] + sequence_to_train_frames[sequence_name].append(frame_entry) + sequence_to_train_frames = dict(sequence_to_train_frames) + test_subset_mapping_set = {tuple(s) for s in subset_mapping["test"]} + + # extend the eval batches / subset mapping with the additional examples + eval_batch_index_out = copy.deepcopy(eval_batch_index) + generator = np.random.default_rng(seed=0) + for batch in eval_batch_index_out: + sequence_name = batch[0][0] + sequence_known_entries = sequence_to_train_frames[sequence_name] + idx_to_add = generator.permutation(len(sequence_known_entries))[ + : self.n_known_frames_for_test + ] + entries_to_add = [sequence_known_entries[a] for a in idx_to_add] + assert all(e in subset_mapping["train"] for e in entries_to_add) + + # extend the eval batch with the known views + batch.extend(entries_to_add) + + # also add these new entries to the test subset mapping + test_subset_mapping_set.update(tuple(e) for e in entries_to_add) + + return eval_batch_index_out, list(test_subset_mapping_set) + + +def get_available_subset_names( + dataset_root: str, + category: str, + path_manager: Optional[PathManager] = None, +) -> List[str]: + """ + Get the available subset names for a given category folder inside a root dataset + folder `dataset_root`. + """ + category_dir = os.path.join(dataset_root, category) + category_dir_exists = ( + (path_manager is not None) and path_manager.isdir(category_dir) + ) or os.path.isdir(category_dir) + if not category_dir_exists: + raise ValueError( + f"Looking for dataset files in {category_dir}. " + + "Please specify a correct dataset_root folder." + ) + + set_list_dir = os.path.join(category_dir, "set_lists") + set_list_jsons = (os.listdir if path_manager is None else path_manager.ls)( + set_list_dir + ) + + return [ + json_file.replace("set_lists_", "").replace(".json", "") + for json_file in set_list_jsons + ] diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/llff_dataset_map_provider.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/llff_dataset_map_provider.py new file mode 100644 index 0000000000000000000000000000000000000000..0c8af9abd14c9f216fe77c9d2727ed193423c936 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/llff_dataset_map_provider.py @@ -0,0 +1,69 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +import numpy as np +import torch +from pytorch3d.implicitron.tools.config import registry + +from .load_llff import load_llff_data + +from .single_sequence_dataset import ( + _interpret_blender_cameras, + SingleSceneDatasetMapProviderBase, +) + + +@registry.register +class LlffDatasetMapProvider(SingleSceneDatasetMapProviderBase): + """ + Provides data for one scene from the LLFF dataset. + + Members: + base_dir: directory holding the data for the scene. + object_name: The name of the scene (e.g. "fern"). This is just used as a label. + It will typically be equal to the name of the directory self.base_dir. + path_manager_factory: Creates path manager which may be used for + interpreting paths. + n_known_frames_for_test: If set, training frames are included in the val + and test datasets, and this many random training frames are added to + each test batch. If not set, test batches each contain just a single + testing frame. + downscale_factor: determines image sizes. + """ + + downscale_factor: int = 4 + + def _load_data(self) -> None: + path_manager = self.path_manager_factory.get() + images, poses, _ = load_llff_data( + self.base_dir, factor=self.downscale_factor, path_manager=path_manager + ) + hwf = poses[0, :3, -1] + poses = poses[:, :3, :4] + + llffhold = 8 + i_test = np.arange(images.shape[0])[::llffhold] + i_test_index = set(i_test.tolist()) + i_train = np.array( + [i for i in np.arange(images.shape[0]) if i not in i_test_index] + ) + i_split = (i_train, i_test, i_test) + H, W, focal = hwf + focal_ndc = 2 * focal / min(H, W) + images = torch.from_numpy(images).permute(0, 3, 1, 2) + poses = torch.from_numpy(poses) + + # pyre-ignore[16] + self.poses = _interpret_blender_cameras(poses, focal_ndc) + # pyre-ignore[16] + self.images = images + # pyre-ignore[16] + self.fg_probabilities = None + # pyre-ignore[16] + self.i_split = i_split diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/load_blender.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/load_blender.py new file mode 100644 index 0000000000000000000000000000000000000000..84f08312d5079c3079f33f30cf16a7ea469721ee --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/load_blender.py @@ -0,0 +1,143 @@ +# @lint-ignore-every LICENSELINT +# Adapted from https://github.com/bmild/nerf/blob/master/load_blender.py +# Copyright (c) 2020 bmild + +# pyre-unsafe +import json +import os + +import numpy as np +import torch +from PIL import Image + + +def translate_by_t_along_z(t): + tform = np.eye(4).astype(np.float32) + tform[2][3] = t + return tform + + +def rotate_by_phi_along_x(phi): + tform = np.eye(4).astype(np.float32) + tform[1, 1] = tform[2, 2] = np.cos(phi) + tform[1, 2] = -np.sin(phi) + tform[2, 1] = -tform[1, 2] + return tform + + +def rotate_by_theta_along_y(theta): + tform = np.eye(4).astype(np.float32) + tform[0, 0] = tform[2, 2] = np.cos(theta) + tform[0, 2] = -np.sin(theta) + tform[2, 0] = -tform[0, 2] + return tform + + +def pose_spherical(theta, phi, radius): + c2w = translate_by_t_along_z(radius) + c2w = rotate_by_phi_along_x(phi / 180.0 * np.pi) @ c2w + c2w = rotate_by_theta_along_y(theta / 180 * np.pi) @ c2w + c2w = np.array([[-1, 0, 0, 0], [0, 0, 1, 0], [0, 1, 0, 0], [0, 0, 0, 1]]) @ c2w + return c2w + + +def _local_path(path_manager, path): + if path_manager is None: + return path + return path_manager.get_local_path(path) + + +def load_blender_data( + basedir, + half_res=False, + testskip=1, + debug=False, + path_manager=None, + focal_length_in_screen_space=False, +): + splits = ["train", "val", "test"] + metas = {} + for s in splits: + path = os.path.join(basedir, f"transforms_{s}.json") + with open(_local_path(path_manager, path)) as fp: + metas[s] = json.load(fp) + + all_imgs = [] + all_poses = [] + counts = [0] + for s in splits: + meta = metas[s] + imgs = [] + poses = [] + if s == "train" or testskip == 0: + skip = 1 + else: + skip = testskip + + for frame in meta["frames"][::skip]: + fname = os.path.join(basedir, frame["file_path"] + ".png") + imgs.append(np.array(Image.open(_local_path(path_manager, fname)))) + poses.append(np.array(frame["transform_matrix"])) + imgs = (np.array(imgs) / 255.0).astype(np.float32) + poses = np.array(poses).astype(np.float32) + counts.append(counts[-1] + imgs.shape[0]) + all_imgs.append(imgs) + all_poses.append(poses) + + i_split = [np.arange(counts[i], counts[i + 1]) for i in range(3)] + + imgs = np.concatenate(all_imgs, 0) + poses = np.concatenate(all_poses, 0) + + H, W = imgs[0].shape[:2] + camera_angle_x = float(meta["camera_angle_x"]) + if focal_length_in_screen_space: + focal = 0.5 * W / np.tan(0.5 * camera_angle_x) + else: + focal = 1 / np.tan(0.5 * camera_angle_x) + + render_poses = torch.stack( + [ + torch.from_numpy(pose_spherical(angle, -30.0, 4.0)) + for angle in np.linspace(-180, 180, 40 + 1)[:-1] + ], + 0, + ) + + # In debug mode, return extremely tiny images + if debug: + import cv2 + + H = H // 32 + W = W // 32 + if focal_length_in_screen_space: + focal = focal / 32.0 + imgs = [ + torch.from_numpy( + cv2.resize(imgs[i], dsize=(25, 25), interpolation=cv2.INTER_AREA) + ) + for i in range(imgs.shape[0]) + ] + imgs = torch.stack(imgs, 0) + poses = torch.from_numpy(poses) + return imgs, poses, render_poses, [H, W, focal], i_split + + if half_res: + import cv2 + + # TODO: resize images using INTER_AREA (cv2) + H = H // 2 + W = W // 2 + if focal_length_in_screen_space: + focal = focal / 2.0 + imgs = [ + torch.from_numpy( + cv2.resize(imgs[i], dsize=(400, 400), interpolation=cv2.INTER_AREA) + ) + for i in range(imgs.shape[0]) + ] + imgs = torch.stack(imgs, 0) + + poses = torch.from_numpy(poses) + + return imgs, poses, render_poses, [H, W, focal], i_split diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/load_llff.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/load_llff.py new file mode 100644 index 0000000000000000000000000000000000000000..d29b650b7e2e4b602e5d3c92c78fb21974ec6ddf --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/load_llff.py @@ -0,0 +1,341 @@ +# @lint-ignore-every LICENSELINT +# Adapted from https://github.com/bmild/nerf/blob/master/load_llff.py +# Copyright (c) 2020 bmild + +# pyre-unsafe +import logging +import os +import warnings + +import numpy as np + +from PIL import Image + + +# Slightly modified version of LLFF data loading code +# see https://github.com/Fyusion/LLFF for original + +logger = logging.getLogger(__name__) + + +def _minify(basedir, path_manager, factors=(), resolutions=()): + needtoload = False + for r in factors: + imgdir = os.path.join(basedir, "images_{}".format(r)) + if not _exists(path_manager, imgdir): + needtoload = True + for r in resolutions: + imgdir = os.path.join(basedir, "images_{}x{}".format(r[1], r[0])) + if not _exists(path_manager, imgdir): + needtoload = True + if not needtoload: + return + assert path_manager is None + + from subprocess import check_output + + imgdir = os.path.join(basedir, "images") + imgs = [os.path.join(imgdir, f) for f in sorted(_ls(path_manager, imgdir))] + imgs = [f for f in imgs if f.endswith("JPG", "jpg", "png", "jpeg", "PNG")] + imgdir_orig = imgdir + + wd = os.getcwd() + + for r in factors + resolutions: + if isinstance(r, int): + name = "images_{}".format(r) + resizearg = "{}%".format(100.0 / r) + else: + name = "images_{}x{}".format(r[1], r[0]) + resizearg = "{}x{}".format(r[1], r[0]) + imgdir = os.path.join(basedir, name) + if os.path.exists(imgdir): + continue + + logger.info(f"Minifying {r}, {basedir}") + + os.makedirs(imgdir) + check_output("cp {}/* {}".format(imgdir_orig, imgdir), shell=True) + + ext = imgs[0].split(".")[-1] + args = " ".join( + ["mogrify", "-resize", resizearg, "-format", "png", "*.{}".format(ext)] + ) + logger.info(args) + os.chdir(imgdir) + check_output(args, shell=True) + os.chdir(wd) + + if ext != "png": + check_output("rm {}/*.{}".format(imgdir, ext), shell=True) + logger.info("Removed duplicates") + logger.info("Done") + + +def _load_data( + basedir, factor=None, width=None, height=None, load_imgs=True, path_manager=None +): + + poses_arr = np.load( + _local_path(path_manager, os.path.join(basedir, "poses_bounds.npy")) + ) + poses = poses_arr[:, :-2].reshape([-1, 3, 5]).transpose([1, 2, 0]) + bds = poses_arr[:, -2:].transpose([1, 0]) + + img0 = [ + os.path.join(basedir, "images", f) + for f in sorted(_ls(path_manager, os.path.join(basedir, "images"))) + if f.endswith("JPG") or f.endswith("jpg") or f.endswith("png") + ][0] + + def imread(f): + return np.array(Image.open(f)) + + sh = imread(_local_path(path_manager, img0)).shape + + sfx = "" + + if factor is not None: + sfx = "_{}".format(factor) + _minify(basedir, path_manager, factors=[factor]) + factor = factor + elif height is not None: + factor = sh[0] / float(height) + width = int(sh[1] / factor) + _minify(basedir, path_manager, resolutions=[[height, width]]) + sfx = "_{}x{}".format(width, height) + elif width is not None: + factor = sh[1] / float(width) + height = int(sh[0] / factor) + _minify(basedir, path_manager, resolutions=[[height, width]]) + sfx = "_{}x{}".format(width, height) + else: + factor = 1 + + imgdir = os.path.join(basedir, "images" + sfx) + if not _exists(path_manager, imgdir): + raise ValueError(f"{imgdir} does not exist, returning") + + imgfiles = [ + _local_path(path_manager, os.path.join(imgdir, f)) + for f in sorted(_ls(path_manager, imgdir)) + if f.endswith("JPG") or f.endswith("jpg") or f.endswith("png") + ] + if poses.shape[-1] != len(imgfiles): + raise ValueError( + "Mismatch between imgs {} and poses {} !!!!".format( + len(imgfiles), poses.shape[-1] + ) + ) + + sh = imread(imgfiles[0]).shape + poses[:2, 4, :] = np.array(sh[:2]).reshape([2, 1]) + poses[2, 4, :] = poses[2, 4, :] * 1.0 / factor + + if not load_imgs: + return poses, bds + + imgs = imgs = [imread(f)[..., :3] / 255.0 for f in imgfiles] + imgs = np.stack(imgs, -1) + + logger.info(f"Loaded image data, shape {imgs.shape}") + return poses, bds, imgs + + +def normalize(x): + denom = np.linalg.norm(x) + if denom < 0.001: + warnings.warn("unsafe normalize()") + return x / denom + + +def viewmatrix(z, up, pos): + vec2 = normalize(z) + vec1_avg = up + vec0 = normalize(np.cross(vec1_avg, vec2)) + vec1 = normalize(np.cross(vec2, vec0)) + m = np.stack([vec0, vec1, vec2, pos], 1) + return m + + +def ptstocam(pts, c2w): + tt = np.matmul(c2w[:3, :3].T, (pts - c2w[:3, 3])[..., np.newaxis])[..., 0] + return tt + + +def poses_avg(poses): + + hwf = poses[0, :3, -1:] + + center = poses[:, :3, 3].mean(0) + vec2 = normalize(poses[:, :3, 2].sum(0)) + up = poses[:, :3, 1].sum(0) + c2w = np.concatenate([viewmatrix(vec2, up, center), hwf], 1) + + return c2w + + +def render_path_spiral(c2w, up, rads, focal, zdelta, zrate, rots, N): + render_poses = [] + rads = np.array(list(rads) + [1.0]) + hwf = c2w[:, 4:5] + + for theta in np.linspace(0.0, 2.0 * np.pi * rots, N + 1)[:-1]: + c = np.dot( + c2w[:3, :4], + np.array([np.cos(theta), -np.sin(theta), -np.sin(theta * zrate), 1.0]) + * rads, + ) + z = normalize(c - np.dot(c2w[:3, :4], np.array([0, 0, -focal, 1.0]))) + render_poses.append(np.concatenate([viewmatrix(z, up, c), hwf], 1)) + return render_poses + + +def recenter_poses(poses): + + poses_ = poses + 0 + bottom = np.reshape([0, 0, 0, 1.0], [1, 4]) + c2w = poses_avg(poses) + c2w = np.concatenate([c2w[:3, :4], bottom], -2) + bottom = np.tile(np.reshape(bottom, [1, 1, 4]), [poses.shape[0], 1, 1]) + poses = np.concatenate([poses[:, :3, :4], bottom], -2) + + poses = np.linalg.inv(c2w) @ poses + poses_[:, :3, :4] = poses[:, :3, :4] + poses = poses_ + return poses + + +def spherify_poses(poses, bds): + def add_row_to_homogenize_transform(p): + r"""Add the last row to homogenize 3 x 4 transformation matrices.""" + return np.concatenate( + [p, np.tile(np.reshape(np.eye(4)[-1, :], [1, 1, 4]), [p.shape[0], 1, 1])], 1 + ) + + # p34_to_44 = lambda p: np.concatenate( + # [p, np.tile(np.reshape(np.eye(4)[-1, :], [1, 1, 4]), [p.shape[0], 1, 1])], 1 + # ) + + p34_to_44 = add_row_to_homogenize_transform + + rays_d = poses[:, :3, 2:3] + rays_o = poses[:, :3, 3:4] + + def min_line_dist(rays_o, rays_d): + A_i = np.eye(3) - rays_d * np.transpose(rays_d, [0, 2, 1]) + b_i = -A_i @ rays_o + pt_mindist = np.squeeze( + -np.linalg.inv((np.transpose(A_i, [0, 2, 1]) @ A_i).mean(0)) @ (b_i).mean(0) + ) + return pt_mindist + + pt_mindist = min_line_dist(rays_o, rays_d) + + center = pt_mindist + up = (poses[:, :3, 3] - center).mean(0) + + vec0 = normalize(up) + vec1 = normalize(np.cross([0.1, 0.2, 0.3], vec0)) + vec2 = normalize(np.cross(vec0, vec1)) + pos = center + c2w = np.stack([vec1, vec2, vec0, pos], 1) + + poses_reset = np.linalg.inv(p34_to_44(c2w[None])) @ p34_to_44(poses[:, :3, :4]) + + rad = np.sqrt(np.mean(np.sum(np.square(poses_reset[:, :3, 3]), -1))) + + sc = 1.0 / rad + poses_reset[:, :3, 3] *= sc + bds *= sc + rad *= sc + + centroid = np.mean(poses_reset[:, :3, 3], 0) + zh = centroid[2] + radcircle = np.sqrt(rad**2 - zh**2) + new_poses = [] + + for th in np.linspace(0.0, 2.0 * np.pi, 120): + + camorigin = np.array([radcircle * np.cos(th), radcircle * np.sin(th), zh]) + up = np.array([0, 0, -1.0]) + + vec2 = normalize(camorigin) + vec0 = normalize(np.cross(vec2, up)) + vec1 = normalize(np.cross(vec2, vec0)) + pos = camorigin + p = np.stack([vec0, vec1, vec2, pos], 1) + + new_poses.append(p) + + new_poses = np.stack(new_poses, 0) + + new_poses = np.concatenate( + [new_poses, np.broadcast_to(poses[0, :3, -1:], new_poses[:, :3, -1:].shape)], -1 + ) + poses_reset = np.concatenate( + [ + poses_reset[:, :3, :4], + np.broadcast_to(poses[0, :3, -1:], poses_reset[:, :3, -1:].shape), + ], + -1, + ) + + return poses_reset, new_poses, bds + + +def _local_path(path_manager, path): + if path_manager is None: + return path + return path_manager.get_local_path(path) + + +def _ls(path_manager, path): + if path_manager is None: + return os.listdir(path) + return path_manager.ls(path) + + +def _exists(path_manager, path): + if path_manager is None: + return os.path.exists(path) + return path_manager.exists(path) + + +def load_llff_data( + basedir, + factor=8, + recenter=True, + bd_factor=0.75, + spherify=False, + path_zflat=False, + path_manager=None, +): + + poses, bds, imgs = _load_data( + basedir, factor=factor, path_manager=path_manager + ) # factor=8 downsamples original imgs by 8x + logger.info(f"Loaded {basedir}, {bds.min()}, {bds.max()}") + + # Correct rotation matrix ordering and move variable dim to axis 0 + poses = np.concatenate([poses[:, 1:2, :], -poses[:, 0:1, :], poses[:, 2:, :]], 1) + poses = np.moveaxis(poses, -1, 0).astype(np.float32) + imgs = np.moveaxis(imgs, -1, 0).astype(np.float32) + images = imgs + bds = np.moveaxis(bds, -1, 0).astype(np.float32) + + # Rescale if bd_factor is provided + sc = 1.0 if bd_factor is None else 1.0 / (bds.min() * bd_factor) + poses[:, :3, 3] *= sc + bds *= sc + + if recenter: + poses = recenter_poses(poses) + + if spherify: + poses, render_poses, bds = spherify_poses(poses, bds) + + images = images.astype(np.float32) + poses = poses.astype(np.float32) + + return images, poses, bds diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/orm_types.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/orm_types.py new file mode 100644 index 0000000000000000000000000000000000000000..2e916021a9a80d48ae8a9741694ca8f5bce38c56 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/orm_types.py @@ -0,0 +1,189 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# This functionality requires SQLAlchemy 2.0 or later. + +import math +import struct +from typing import Optional, Tuple + +import numpy as np + +from pytorch3d.implicitron.dataset.types import ( + DepthAnnotation, + ImageAnnotation, + MaskAnnotation, + PointCloudAnnotation, + VideoAnnotation, + ViewpointAnnotation, +) + +from sqlalchemy import LargeBinary +from sqlalchemy.orm import ( + composite, + DeclarativeBase, + Mapped, + mapped_column, + MappedAsDataclass, +) +from sqlalchemy.types import TypeDecorator + + +# these produce policies to serialize structured types to blobs +def ArrayTypeFactory(shape=None): + if shape is None: + + class VariableShapeNumpyArrayType(TypeDecorator): + impl = LargeBinary + + def process_bind_param(self, value, dialect): + if value is None: + return None + + ndim_bytes = np.int32(value.ndim).tobytes() + shape_bytes = np.array(value.shape, dtype=np.int64).tobytes() + value_bytes = value.astype(np.float32).tobytes() + return ndim_bytes + shape_bytes + value_bytes + + def process_result_value(self, value, dialect): + if value is None: + return None + + ndim = np.frombuffer(value[:4], dtype=np.int32)[0] + value_start = 4 + 8 * ndim + shape = np.frombuffer(value[4:value_start], dtype=np.int64) + assert shape.shape == (ndim,) + return np.frombuffer(value[value_start:], dtype=np.float32).reshape( + shape + ) + + return VariableShapeNumpyArrayType + + class NumpyArrayType(TypeDecorator): + impl = LargeBinary + + def process_bind_param(self, value, dialect): + if value is not None: + if value.shape != shape: + raise ValueError(f"Passed an array of wrong shape: {value.shape}") + return value.astype(np.float32).tobytes() + return None + + def process_result_value(self, value, dialect): + if value is not None: + return np.frombuffer(value, dtype=np.float32).reshape(shape) + return None + + return NumpyArrayType + + +def TupleTypeFactory(dtype=float, shape: Tuple[int, ...] = (2,)): + format_symbol = { + float: "f", # float32 + int: "i", # int32 + }[dtype] + + class TupleType(TypeDecorator): + impl = LargeBinary + _format = format_symbol * math.prod(shape) + + def process_bind_param(self, value, _): + if value is None: + return None + + if len(shape) > 1: + value = np.array(value, dtype=dtype).reshape(-1) + + return struct.pack(TupleType._format, *value) + + def process_result_value(self, value, _): + if value is None: + return None + + loaded = struct.unpack(TupleType._format, value) + if len(shape) > 1: + loaded = _rec_totuple( + np.array(loaded, dtype=dtype).reshape(shape).tolist() + ) + + return loaded + + return TupleType + + +def _rec_totuple(t): + if isinstance(t, list): + return tuple(_rec_totuple(x) for x in t) + + return t + + +class Base(MappedAsDataclass, DeclarativeBase): + """subclasses will be converted to dataclasses""" + + +class SqlFrameAnnotation(Base): + __tablename__ = "frame_annots" + + sequence_name: Mapped[str] = mapped_column(primary_key=True) + frame_number: Mapped[int] = mapped_column(primary_key=True) + frame_timestamp: Mapped[float] = mapped_column(index=True) + + image: Mapped[ImageAnnotation] = composite( + mapped_column("_image_path"), + mapped_column("_image_size", TupleTypeFactory(int)), + ) + + depth: Mapped[DepthAnnotation] = composite( + mapped_column("_depth_path", nullable=True), + mapped_column("_depth_scale_adjustment", nullable=True), + mapped_column("_depth_mask_path", nullable=True), + ) + + mask: Mapped[MaskAnnotation] = composite( + mapped_column("_mask_path", nullable=True), + mapped_column("_mask_mass", index=True, nullable=True), + mapped_column( + "_mask_bounding_box_xywh", + TupleTypeFactory(float, shape=(4,)), + nullable=True, + ), + ) + + viewpoint: Mapped[ViewpointAnnotation] = composite( + mapped_column( + "_viewpoint_R", TupleTypeFactory(float, shape=(3, 3)), nullable=True + ), + mapped_column( + "_viewpoint_T", TupleTypeFactory(float, shape=(3,)), nullable=True + ), + mapped_column( + "_viewpoint_focal_length", TupleTypeFactory(float), nullable=True + ), + mapped_column( + "_viewpoint_principal_point", TupleTypeFactory(float), nullable=True + ), + mapped_column("_viewpoint_intrinsics_format", nullable=True), + ) + + +class SqlSequenceAnnotation(Base): + __tablename__ = "sequence_annots" + + sequence_name: Mapped[str] = mapped_column(primary_key=True) + category: Mapped[str] = mapped_column(index=True) + + video: Mapped[VideoAnnotation] = composite( + mapped_column("_video_path", nullable=True), + mapped_column("_video_length", nullable=True), + ) + point_cloud: Mapped[PointCloudAnnotation] = composite( + mapped_column("_point_cloud_path", nullable=True), + mapped_column("_point_cloud_quality_score", nullable=True), + mapped_column("_point_cloud_n_points", nullable=True), + ) + # the bigger the better + viewpoint_quality_score: Mapped[Optional[float]] = mapped_column() diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/rendered_mesh_dataset_map_provider.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/rendered_mesh_dataset_map_provider.py new file mode 100644 index 0000000000000000000000000000000000000000..56d86f98a246de6556215b4676dbabbe27e8595f --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/rendered_mesh_dataset_map_provider.py @@ -0,0 +1,217 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from os.path import dirname, join, realpath +from typing import Optional, Tuple + +import torch +from pytorch3d.implicitron.tools.config import registry, run_auto_creation +from pytorch3d.io import IO +from pytorch3d.renderer import ( + AmbientLights, + BlendParams, + CamerasBase, + FoVPerspectiveCameras, + HardPhongShader, + look_at_view_transform, + MeshRasterizer, + MeshRendererWithFragments, + PointLights, + RasterizationSettings, +) +from pytorch3d.structures.meshes import Meshes + +from .dataset_map_provider import DatasetMap, DatasetMapProviderBase, PathManagerFactory +from .single_sequence_dataset import SingleSceneDataset +from .utils import DATASET_TYPE_KNOWN + + +@registry.register +class RenderedMeshDatasetMapProvider(DatasetMapProviderBase): # pyre-ignore [13] + """ + A simple single-scene dataset based on PyTorch3D renders of a mesh. + Provides `num_views` renders of the mesh as train, with no val + and test. The renders are generated from viewpoints sampled at uniformly + distributed azimuth intervals. The elevation is kept constant so that the + camera's vertical position coincides with the equator. + + By default, uses Keenan Crane's cow model, and the camera locations are + set to make sense for that. + + Although the rendering used to generate this dataset will use a GPU + if one is available, the data it produces is on the CPU just like + the data returned by implicitron's other dataset map providers. + This is because both datasets and models can be large, so implicitron's + training loop expects data on the CPU and only moves + what it needs to the device. + + For a more detailed explanation of this code, please refer to the + docs/tutorials/fit_textured_mesh.ipynb notebook. + + Members: + num_views: The number of generated renders. + data_file: The folder that contains the mesh file. By default, finds + the cow mesh in the same repo as this code. + azimuth_range: number of degrees on each side of the start position to + take samples + distance: distance from camera centres to the origin. + resolution: the common height and width of the output images. + use_point_light: whether to use a particular point light as opposed + to ambient white. + gpu_idx: which gpu to use for rendering the mesh. + path_manager_factory: (Optional) An object that generates an instance of + PathManager that can translate provided file paths. + path_manager_factory_class_type: The class type of `path_manager_factory`. + """ + + num_views: int = 40 + data_file: Optional[str] = None + azimuth_range: float = 180 + distance: float = 2.7 + resolution: int = 128 + use_point_light: bool = True + gpu_idx: Optional[int] = 0 + path_manager_factory: PathManagerFactory + path_manager_factory_class_type: str = "PathManagerFactory" + + def get_dataset_map(self) -> DatasetMap: + # pyre-ignore[16] + return DatasetMap(train=self.train_dataset, val=None, test=None) + + def get_all_train_cameras(self) -> CamerasBase: + # pyre-ignore[16] + return self.poses + + def __post_init__(self) -> None: + super().__init__() + run_auto_creation(self) + if torch.cuda.is_available() and self.gpu_idx is not None: + device = torch.device(f"cuda:{self.gpu_idx}") + else: + device = torch.device("cpu") + if self.data_file is None: + data_file = join( + dirname(dirname(dirname(dirname(realpath(__file__))))), + "docs", + "tutorials", + "data", + "cow_mesh", + "cow.obj", + ) + else: + data_file = self.data_file + io = IO(path_manager=self.path_manager_factory.get()) + mesh = io.load_mesh(data_file, device=device) + poses, images, masks = _generate_cow_renders( + num_views=self.num_views, + mesh=mesh, + azimuth_range=self.azimuth_range, + distance=self.distance, + resolution=self.resolution, + device=device, + use_point_light=self.use_point_light, + ) + # pyre-ignore[16] + self.poses = poses.cpu() + # pyre-ignore[16] + self.train_dataset = SingleSceneDataset( # pyre-ignore[28] + object_name="cow", + images=list(images.permute(0, 3, 1, 2).cpu()), + fg_probabilities=list(masks[:, None].cpu()), + poses=[self.poses[i] for i in range(len(poses))], + frame_types=[DATASET_TYPE_KNOWN] * len(poses), + eval_batches=None, + ) + + +@torch.no_grad() +def _generate_cow_renders( + *, + num_views: int, + mesh: Meshes, + azimuth_range: float, + distance: float, + resolution: int, + device: torch.device, + use_point_light: bool, +) -> Tuple[CamerasBase, torch.Tensor, torch.Tensor]: + """ + Returns: + cameras: A batch of `num_views` `FoVPerspectiveCameras` from which the + images are rendered. + images: A tensor of shape `(num_views, height, width, 3)` containing + the rendered images. + silhouettes: A tensor of shape `(num_views, height, width)` containing + the rendered silhouettes. + """ + + # Load obj file + + # We scale normalize and center the target mesh to fit in a sphere of radius 1 + # centered at (0,0,0). (scale, center) will be used to bring the predicted mesh + # to its original center and scale. Note that normalizing the target mesh, + # speeds up the optimization but is not necessary! + verts = mesh.verts_packed() + N = verts.shape[0] + center = verts.mean(0) + scale = max((verts - center).abs().max(0)[0]) + mesh.offset_verts_(-(center.expand(N, 3))) + mesh.scale_verts_((1.0 / float(scale))) + + # Get a batch of viewing angles. + elev = torch.linspace(0, 0, num_views) # keep constant + azim = torch.linspace(-azimuth_range, azimuth_range, num_views) + 180.0 + + # Place a point light in front of the object. As mentioned above, the front of + # the cow is facing the -z direction. + if use_point_light: + lights = PointLights(device=device, location=[[0.0, 0.0, -3.0]]) + else: + lights = AmbientLights(device=device) + + # Initialize a perspective camera that represents a batch of different + # viewing angles. All the cameras helper methods support mixed type inputs and + # broadcasting. So we can view the camera from a fixed distance, and + # then specify elevation and azimuth angles for each viewpoint as tensors. + R, T = look_at_view_transform(dist=distance, elev=elev, azim=azim) + cameras = FoVPerspectiveCameras(device=device, R=R, T=T) + + # Define the settings for rasterization and shading. + # As we are rendering images for visualization + # purposes only we will set faces_per_pixel=1 and blur_radius=0.0. Refer to + # rasterize_meshes.py for explanations of these parameters. We also leave + # bin_size and max_faces_per_bin to their default values of None, which sets + # their values using heuristics and ensures that the faster coarse-to-fine + # rasterization method is used. Refer to docs/notes/renderer.md for an + # explanation of the difference between naive and coarse-to-fine rasterization. + raster_settings = RasterizationSettings( + image_size=resolution, blur_radius=0.0, faces_per_pixel=1 + ) + + # Create a Phong renderer by composing a rasterizer and a shader. The textured + # Phong shader will interpolate the texture uv coordinates for each vertex, + # sample from a texture image and apply the Phong lighting model + blend_params = BlendParams(sigma=1e-4, gamma=1e-4, background_color=(0.0, 0.0, 0.0)) + rasterizer_type = MeshRasterizer + renderer = MeshRendererWithFragments( + rasterizer=rasterizer_type(cameras=cameras, raster_settings=raster_settings), + shader=HardPhongShader( + device=device, cameras=cameras, lights=lights, blend_params=blend_params + ), + ) + + # Create a batch of meshes by repeating the cow mesh and associated textures. + # Meshes has a useful `extend` method which allows us do this very easily. + # This also extends the textures. + meshes = mesh.extend(num_views) + + # Render the cow mesh from each viewing angle + target_images, fragments = renderer(meshes, cameras=cameras, lights=lights) + silhouette_binary = (fragments.pix_to_face[..., 0] >= 0).float() + + return cameras, target_images[..., :3], silhouette_binary diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/scene_batch_sampler.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/scene_batch_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..2ce7e27176e0bfc9a47a305d025ad837aae66ab8 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/scene_batch_sampler.py @@ -0,0 +1,216 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +import warnings +from collections import Counter +from dataclasses import dataclass, field +from typing import Dict, Iterable, Iterator, List, Sequence, Tuple + +import numpy as np +from torch.utils.data.sampler import Sampler + +from .dataset_base import DatasetBase + + +@dataclass(eq=False) # TODO: do we need this if not init from config? +class SceneBatchSampler(Sampler[List[int]]): + """ + A class for sampling training batches with a controlled composition + of sequences. + """ + + dataset: DatasetBase + batch_size: int + num_batches: int + # the sampler first samples a random element k from this list and then + # takes k random frames per sequence + images_per_seq_options: Sequence[int] + + # if True, will sample a contiguous interval of frames in the sequence + # it first finds the connected segments within the sequence of sufficient length, + # then samples a random pivot element among them and ideally uses it as a middle + # of the temporal window, shifting the borders where necessary. + # This strategy mitigates the bias against shorter segments and their boundaries. + sample_consecutive_frames: bool = False + # if a number > 0, then used to define the maximum difference in frame_number + # of neighbouring frames when forming connected segments; otherwise the whole + # sequence is considered a segment regardless of frame numbers + consecutive_frames_max_gap: int = 0 + # same but for timestamps if they are available + consecutive_frames_max_gap_seconds: float = 0.1 + + # if True, the sampler first reads from the dataset the mapping between + # sequence names and their categories. + # During batch sampling, the sampler ensures uniform distribution over the categories + # of the sampled sequences. + category_aware: bool = True + + seq_names: List[str] = field(init=False) + + category_to_sequence_names: Dict[str, List[str]] = field(init=False) + categories: List[str] = field(init=False) + + def __post_init__(self) -> None: + if self.batch_size <= 0: + raise ValueError( + "batch_size should be a positive integral value, " + f"but got batch_size={self.batch_size}" + ) + + if len(self.images_per_seq_options) < 1: + raise ValueError("n_per_seq_posibilities list cannot be empty") + + self.seq_names = list(self.dataset.sequence_names()) + + if self.category_aware: + self.category_to_sequence_names = self.dataset.category_to_sequence_names() + self.categories = list(self.category_to_sequence_names.keys()) + + def __len__(self) -> int: + return self.num_batches + + def __iter__(self) -> Iterator[List[int]]: + for batch_idx in range(len(self)): + batch = self._sample_batch(batch_idx) + yield batch + + def _sample_batch(self, batch_idx) -> List[int]: + n_per_seq = np.random.choice(self.images_per_seq_options) + n_seqs = -(-self.batch_size // n_per_seq) # round up + + if self.category_aware: + # first sample categories at random, these can be repeated in the batch + chosen_cat = _capped_random_choice(self.categories, n_seqs, replace=True) + # then randomly sample a set of unique sequences within each category + chosen_seq = [] + for cat, n_per_category in Counter(chosen_cat).items(): + category_chosen_seq = _capped_random_choice( + self.category_to_sequence_names[cat], + n_per_category, + replace=False, + ) + chosen_seq.extend([str(s) for s in category_chosen_seq]) + else: + chosen_seq = _capped_random_choice( + self.seq_names, + n_seqs, + replace=False, + ) + + if self.sample_consecutive_frames: + frame_idx = [] + for seq in chosen_seq: + segment_index = self._build_segment_index(seq, n_per_seq) + + segment, idx = segment_index[np.random.randint(len(segment_index))] + if len(segment) <= n_per_seq: + frame_idx.append(segment) + else: + start = np.clip(idx - n_per_seq // 2, 0, len(segment) - n_per_seq) + frame_idx.append(segment[start : start + n_per_seq]) + + else: + frame_idx = [ + _capped_random_choice( + list(self.dataset.sequence_indices_in_order(seq)), + n_per_seq, + replace=False, + ) + for seq in chosen_seq + ] + frame_idx = np.concatenate(frame_idx)[: self.batch_size].tolist() + if len(frame_idx) < self.batch_size: + warnings.warn( + "Batch size smaller than self.batch_size!" + + " (This is fine for experiments with a single scene and viewpooling)" + ) + return frame_idx + + def _build_segment_index(self, seq: str, size: int) -> List[Tuple[List[int], int]]: + """ + Returns a list of (segment, index) tuples, one per eligible frame, where + segment is a list of frame indices in the contiguous segment the frame + belongs to index is the frame's index within that segment. + Segment references are repeated but the memory is shared. + """ + if ( + self.consecutive_frames_max_gap > 0 + or self.consecutive_frames_max_gap_seconds > 0.0 + ): + segments = self._split_to_segments( + self.dataset.sequence_frames_in_order(seq) + ) + segments = _cull_short_segments(segments, size) + if not segments: + raise AssertionError("Empty segments after culling") + else: + segments = [list(self.dataset.sequence_indices_in_order(seq))] + + # build an index of segment for random selection of a pivot frame + segment_index = [ + (segment, i) for segment in segments for i in range(len(segment)) + ] + + return segment_index + + def _split_to_segments( + self, sequence_timestamps: Iterable[Tuple[float, int, int]] + ) -> List[List[int]]: + if ( + self.consecutive_frames_max_gap <= 0 + and self.consecutive_frames_max_gap_seconds <= 0.0 + ): + raise AssertionError("This function is only needed for non-trivial max_gap") + + segments = [] + last_no = -self.consecutive_frames_max_gap - 1 # will trigger a new segment + last_ts = -self.consecutive_frames_max_gap_seconds - 1.0 + for ts, no, idx in sequence_timestamps: + if ts <= 0.0 and no <= last_no: + raise AssertionError( + "Sequence frames are not ordered while timestamps are not given" + ) + + if ( + no - last_no > self.consecutive_frames_max_gap > 0 + or ts - last_ts > self.consecutive_frames_max_gap_seconds > 0.0 + ): # new group + segments.append([idx]) + else: + segments[-1].append(idx) + + last_no = no + last_ts = ts + + return segments + + +def _cull_short_segments(segments: List[List[int]], min_size: int) -> List[List[int]]: + lengths = [(len(segment), segment) for segment in segments] + max_len, longest_segment = max(lengths) + + if max_len < min_size: + return [longest_segment] + + return [segment for segment in segments if len(segment) >= min_size] + + +def _capped_random_choice(x, size, replace: bool = True): + """ + if replace==True + randomly chooses from x `size` elements without replacement if len(x)>size + else allows replacement and selects `size` elements again. + if replace==False + randomly chooses from x `min(len(x), size)` elements without replacement + """ + len_x = x if isinstance(x, int) else len(x) + if replace: + return np.random.choice(x, size=size, replace=len_x < size) + else: + return np.random.choice(x, size=min(size, len_x), replace=False) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/single_sequence_dataset.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/single_sequence_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..d99b44e0f2fdcdb9eade0c195a7980c95a1ad903 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/single_sequence_dataset.py @@ -0,0 +1,208 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +# This file defines a base class for dataset map providers which +# provide data for a single scene. + +from dataclasses import field +from typing import Iterable, Iterator, List, Optional, Sequence, Tuple + +import numpy as np +import torch +from pytorch3d.implicitron.tools.config import ( + Configurable, + expand_args_fields, + run_auto_creation, +) +from pytorch3d.renderer import CamerasBase, join_cameras_as_batch, PerspectiveCameras + +from .dataset_base import DatasetBase +from .dataset_map_provider import DatasetMap, DatasetMapProviderBase, PathManagerFactory +from .frame_data import FrameData +from .utils import DATASET_TYPE_KNOWN, DATASET_TYPE_UNKNOWN + +_SINGLE_SEQUENCE_NAME: str = "one_sequence" + + +@expand_args_fields +class SingleSceneDataset(DatasetBase, Configurable): + """ + A dataset from images from a single scene. + """ + + images: List[torch.Tensor] = field() + fg_probabilities: Optional[List[torch.Tensor]] = field() + poses: List[PerspectiveCameras] = field() + object_name: str = field() + frame_types: List[str] = field() + eval_batches: Optional[List[List[int]]] = field() + + def sequence_names(self) -> Iterable[str]: + return [_SINGLE_SEQUENCE_NAME] + + def __len__(self) -> int: + return len(self.poses) + + def sequence_frames_in_order( + self, seq_name: str, subset_filter: Optional[Sequence[str]] = None + ) -> Iterator[Tuple[float, int, int]]: + for i in range(len(self)): + if subset_filter is None or self.frame_types[i] in subset_filter: + yield 0.0, i, i + + def __getitem__(self, index) -> FrameData: + if index >= len(self): + raise IndexError(f"index {index} out of range {len(self)}") + image = self.images[index] + pose = self.poses[index] + frame_type = self.frame_types[index] + fg_probability = ( + None if self.fg_probabilities is None else self.fg_probabilities[index] + ) + + frame_data = FrameData( + frame_number=index, + sequence_name=_SINGLE_SEQUENCE_NAME, + sequence_category=self.object_name, + camera=pose, + # pyre-ignore + image_size_hw=torch.tensor(image.shape[1:], dtype=torch.long), + image_rgb=image, + fg_probability=fg_probability, + frame_type=frame_type, + ) + return frame_data + + def get_eval_batches(self) -> Optional[List[List[int]]]: + return self.eval_batches + + +# pyre-fixme[13]: Uninitialized attribute +class SingleSceneDatasetMapProviderBase(DatasetMapProviderBase): + """ + Base for provider of data for one scene from LLFF or blender datasets. + + Members: + base_dir: directory holding the data for the scene. + object_name: The name of the scene (e.g. "lego"). This is just used as a label. + It will typically be equal to the name of the directory self.base_dir. + path_manager_factory: Creates path manager which may be used for + interpreting paths. + n_known_frames_for_test: If set, training frames are included in the val + and test datasets, and this many random training frames are added to + each test batch. If not set, test batches each contain just a single + testing frame. + """ + + base_dir: str + object_name: str + path_manager_factory: PathManagerFactory + path_manager_factory_class_type: str = "PathManagerFactory" + n_known_frames_for_test: Optional[int] = None + + def __post_init__(self) -> None: + run_auto_creation(self) + self._load_data() + + def _load_data(self) -> None: + # This must be defined by each subclass, + # and should set the following on self. + # - poses: a list of length-1 camera objects + # - images: [N, 3, H, W] tensor of rgb images - floats in [0,1] + # - fg_probabilities: None or [N, 1, H, W] of floats in [0,1] + # - splits: List[List[int]] of indices for train/val/test subsets. + raise NotImplementedError() + + def _get_dataset( + self, split_idx: int, frame_type: str, set_eval_batches: bool = False + ) -> SingleSceneDataset: + # pyre-ignore[16] + split = self.i_split[split_idx] + frame_types = [frame_type] * len(split) + fg_probabilities = ( + None + # pyre-ignore[16] + if self.fg_probabilities is None + else self.fg_probabilities[split] + ) + eval_batches = [[i] for i in range(len(split))] + if split_idx != 0 and self.n_known_frames_for_test is not None: + train_split = self.i_split[0] + if set_eval_batches: + generator = np.random.default_rng(seed=0) + for batch in eval_batches: + # using permutation so that changes to n_known_frames_for_test + # result in consistent batches. + to_add = generator.permutation(len(train_split))[ + : self.n_known_frames_for_test + ] + batch.extend((to_add + len(split)).tolist()) + split = np.concatenate([split, train_split]) + frame_types.extend([DATASET_TYPE_KNOWN] * len(train_split)) + + # pyre-ignore[28] + return SingleSceneDataset( + object_name=self.object_name, + # pyre-ignore[16] + images=self.images[split], + fg_probabilities=fg_probabilities, + # pyre-ignore[16] + poses=[self.poses[i] for i in split], + frame_types=frame_types, + eval_batches=eval_batches if set_eval_batches else None, + ) + + def get_dataset_map(self) -> DatasetMap: + return DatasetMap( + train=self._get_dataset(0, DATASET_TYPE_KNOWN), + val=self._get_dataset(1, DATASET_TYPE_UNKNOWN), + test=self._get_dataset(2, DATASET_TYPE_UNKNOWN, True), + ) + + def get_all_train_cameras(self) -> Optional[CamerasBase]: + # pyre-ignore[16] + cameras = [self.poses[i] for i in self.i_split[0]] + return join_cameras_as_batch(cameras) + + +def _interpret_blender_cameras( + poses: torch.Tensor, focal: float +) -> List[PerspectiveCameras]: + """ + Convert 4x4 matrices representing cameras in blender format + to PyTorch3D format. + + Args: + poses: N x 3 x 4 camera matrices + focal: ndc space focal length + """ + pose_target_cameras = [] + for pose_target in poses: + pose_target = pose_target[:3, :4] + mtx = torch.eye(4, dtype=pose_target.dtype) + mtx[:3, :3] = pose_target[:3, :3].t() + mtx[3, :3] = pose_target[:, 3] + mtx = mtx.inverse() + + # flip the XZ coordinates. + mtx[:, [0, 2]] *= -1.0 + + Rpt3, Tpt3 = mtx[:, :3].split([3, 1], dim=0) + + focal_length_pt3 = torch.FloatTensor([[focal, focal]]) + principal_point_pt3 = torch.FloatTensor([[0.0, 0.0]]) + + cameras = PerspectiveCameras( + focal_length=focal_length_pt3, + principal_point=principal_point_pt3, + R=Rpt3[None], + T=Tpt3, + ) + pose_target_cameras.append(cameras) + return pose_target_cameras diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/sql_dataset.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/sql_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..470f5a95bf100595659918ee979d8e350aa71480 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/sql_dataset.py @@ -0,0 +1,768 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import hashlib +import json +import logging +import os +from dataclasses import dataclass +from typing import ( + Any, + ClassVar, + Dict, + Iterable, + Iterator, + List, + Optional, + Sequence, + Tuple, + Type, + Union, +) + +import numpy as np +import pandas as pd +import sqlalchemy as sa +import torch +from pytorch3d.implicitron.dataset.dataset_base import DatasetBase + +from pytorch3d.implicitron.dataset.frame_data import ( # noqa + FrameData, + FrameDataBuilder, + FrameDataBuilderBase, +) +from pytorch3d.implicitron.tools.config import ( + registry, + ReplaceableBase, + run_auto_creation, +) +from sqlalchemy.orm import Session + +from .orm_types import SqlFrameAnnotation, SqlSequenceAnnotation + + +logger = logging.getLogger(__name__) + + +_SET_LISTS_TABLE: str = "set_lists" + + +@registry.register +class SqlIndexDataset(DatasetBase, ReplaceableBase): # pyre-ignore + """ + A dataset with annotations stored as SQLite tables. This is an index-based dataset. + The length is returned after all sequence and frame filters are applied (see param + definitions below). Indices can either be ordinal in [0, len), or pairs of + (sequence_name, frame_number); with the performance of `dataset[i]` and + `dataset[sequence_name, frame_number]` being same. A faster way to get metadata only + (without blobs) is `dataset.meta[idx]` indexing; it requires box_crop==False. + With ordinal indexing, the sequences are NOT guaranteed to span contiguous index + ranges, and frame numbers are NOT guaranteed to be increasing within a sequence. + Sequence-aware batch samplers have to use `sequence_[frames|indices]_in_order` + iterators, which are efficient. + + This functionality requires SQLAlchemy 2.0 or later. + + Metadata-related args: + sqlite_metadata_file: A SQLite file containing frame and sequence annotation + tables (mapping to SqlFrameAnnotation and SqlSequenceAnnotation, + respectively). + dataset_root: A root directory to look for images, masks, etc. It can be + alternatively set in `frame_data_builder` args, but this takes precedence. + subset_lists_file: A JSON/sqlite file containing the lists of frames + corresponding to different subsets (e.g. train/val/test) of the dataset; + format: {subset: [(sequence_name, frame_id, file_path)]}. All entries + must be present in frame_annotation metadata table. + path_manager: a facade for non-POSIX filesystems. + subsets: Restrict frames/sequences only to the given list of subsets + as defined in subset_lists_file (see above). Applied before all other + filters. + remove_empty_masks: Removes the frames with no active foreground pixels + in the segmentation mask (needs frame_annotation.mask.mass to be set; + null values are retained). + pick_frames_sql_clause: SQL WHERE clause to constrain frame annotations + NOTE: This is a potential security risk! The string is passed to the SQL + engine verbatim. Don’t expose it to end users of your application! + pick_categories: Restrict the dataset to the given list of categories. + pick_sequences: A Sequence of sequence names to restrict the dataset to. + exclude_sequences: A Sequence of the names of the sequences to exclude. + limit_sequences_per_category_to: Limit the dataset to the first up to N + sequences within each category (applies after all other sequence filters + but before `limit_sequences_to`). + limit_sequences_to: Limit the dataset to the first `limit_sequences_to` + sequences (after other sequence filters have been applied but before + frame-based filters). + limit_to: Limit the dataset to the first #limit_to frames (after other + filters have been applied, except n_frames_per_sequence). + n_frames_per_sequence: If > 0, randomly samples `n_frames_per_sequence` + frames in each sequences uniformly without replacement if it has + more frames than that; applied after other frame-level filters. + seed: The seed of the random generator sampling `n_frames_per_sequence` + random frames per sequence. + """ + + frame_annotations_type: ClassVar[Type[SqlFrameAnnotation]] = SqlFrameAnnotation + + sqlite_metadata_file: str = "" + dataset_root: Optional[str] = None + subset_lists_file: str = "" + eval_batches_file: Optional[str] = None + path_manager: Any = None + subsets: Optional[List[str]] = None + remove_empty_masks: bool = True + pick_frames_sql_clause: Optional[str] = None + pick_categories: Tuple[str, ...] = () + + pick_sequences: Tuple[str, ...] = () + exclude_sequences: Tuple[str, ...] = () + limit_sequences_per_category_to: int = 0 + limit_sequences_to: int = 0 + limit_to: int = 0 + n_frames_per_sequence: int = -1 + seed: int = 0 + remove_empty_masks_poll_whole_table_threshold: int = 300_000 + # we set it manually in the constructor + # _index: pd.DataFrame = field(init=False) + + frame_data_builder: FrameDataBuilderBase + frame_data_builder_class_type: str = "FrameDataBuilder" + + def __post_init__(self) -> None: + if sa.__version__ < "2.0": + raise ImportError("This class requires SQL Alchemy 2.0 or later") + + if not self.sqlite_metadata_file: + raise ValueError("sqlite_metadata_file must be set") + + if self.dataset_root: + frame_builder_type = self.frame_data_builder_class_type + getattr(self, f"frame_data_builder_{frame_builder_type}_args")[ + "dataset_root" + ] = self.dataset_root + + run_auto_creation(self) + self.frame_data_builder.path_manager = self.path_manager + + # pyre-ignore # NOTE: sqlite-specific args (read-only mode). + self._sql_engine = sa.create_engine( + f"sqlite:///file:{self.sqlite_metadata_file}?mode=ro&uri=true" + ) + + sequences = self._get_filtered_sequences_if_any() + + if self.subsets: + index = self._build_index_from_subset_lists(sequences) + else: + # TODO: if self.subset_lists_file and not self.subsets, it might be faster to + # still use the concatenated lists, assuming they cover the whole dataset + index = self._build_index_from_db(sequences) + + if self.n_frames_per_sequence >= 0: + index = self._stratified_sample_index(index) + + if len(index) == 0: + raise ValueError(f"There are no frames in the subsets: {self.subsets}!") + + self._index = index.set_index(["sequence_name", "frame_number"]) # pyre-ignore + + self.eval_batches = None # pyre-ignore + if self.eval_batches_file: + self.eval_batches = self._load_filter_eval_batches() + + logger.info(str(self)) + + def __len__(self) -> int: + # pyre-ignore[16] + return len(self._index) + + def __getitem__(self, frame_idx: Union[int, Tuple[str, int]]) -> FrameData: + """ + Fetches FrameData by either iloc in the index or by (sequence, frame_no) pair + """ + return self._get_item(frame_idx, True) + + @property + def meta(self): + """ + Allows accessing metadata only without loading blobs using `dataset.meta[idx]`. + Requires box_crop==False, since in that case, cameras cannot be adjusted + without loading masks. + + Returns: + FrameData objects with blob fields like `image_rgb` set to None. + + Raises: + ValueError if dataset.box_crop is set. + """ + return SqlIndexDataset._MetadataAccessor(self) + + @dataclass + class _MetadataAccessor: + dataset: "SqlIndexDataset" + + def __getitem__(self, frame_idx: Union[int, Tuple[str, int]]) -> FrameData: + return self.dataset._get_item(frame_idx, False) + + def _get_item( + self, frame_idx: Union[int, Tuple[str, int]], load_blobs: bool = True + ) -> FrameData: + if isinstance(frame_idx, int): + if frame_idx >= len(self._index): + raise IndexError(f"index {frame_idx} out of range {len(self._index)}") + + seq, frame = self._index.index[frame_idx] + else: + seq, frame, *rest = frame_idx + if isinstance(frame, torch.LongTensor): + frame = frame.item() + + if (seq, frame) not in self._index.index: + raise IndexError( + f"Sequence-frame index {frame_idx} not found; was it filtered out?" + ) + + if rest and rest[0] != self._index.loc[(seq, frame), "_image_path"]: + raise IndexError(f"Non-matching image path in {frame_idx}.") + + stmt = sa.select(self.frame_annotations_type).where( + self.frame_annotations_type.sequence_name == seq, + self.frame_annotations_type.frame_number + == int(frame), # cast from np.int64 + ) + seq_stmt = sa.select(SqlSequenceAnnotation).where( + SqlSequenceAnnotation.sequence_name == seq + ) + with Session(self._sql_engine) as session: + entry = session.scalars(stmt).one() + seq_metadata = session.scalars(seq_stmt).one() + + assert entry.image.path == self._index.loc[(seq, frame), "_image_path"] + + frame_data = self.frame_data_builder.build( + entry, seq_metadata, load_blobs=load_blobs + ) + + # The rest of the fields are optional + frame_data.frame_type = self._get_frame_type(entry) + return frame_data + + def __str__(self) -> str: + # pyre-ignore[16] + return f"SqlIndexDataset #frames={len(self._index)}" + + def sequence_names(self) -> Iterable[str]: + """Returns an iterator over sequence names in the dataset.""" + return self._index.index.unique("sequence_name") + + # override + def category_to_sequence_names(self) -> Dict[str, List[str]]: + stmt = sa.select( + SqlSequenceAnnotation.category, SqlSequenceAnnotation.sequence_name + ).where( # we limit results to sequences that have frames after all filters + SqlSequenceAnnotation.sequence_name.in_(self.sequence_names()) + ) + with self._sql_engine.connect() as connection: + cat_to_seqs = pd.read_sql(stmt, connection) + + return cat_to_seqs.groupby("category")["sequence_name"].apply(list).to_dict() + + # override + def get_frame_numbers_and_timestamps( + self, idxs: Sequence[int], subset_filter: Optional[Sequence[str]] = None + ) -> List[Tuple[int, float]]: + """ + Implements the DatasetBase method. + + NOTE: Avoid this function as there are more efficient alternatives such as + querying `dataset[idx]` directly or getting all sequence frames with + `sequence_[frames|indices]_in_order`. + + Return the index and timestamp in their videos of the frames whose + indices are given in `idxs`. They need to belong to the same sequence! + If timestamps are absent, they are replaced with zeros. + This is used for letting SceneBatchSampler identify consecutive + frames. + + Args: + idxs: a sequence int frame index in the dataset (it can be a slice) + subset_filter: must remain None + + Returns: + list of tuples of + - frame index in video + - timestamp of frame in video, coalesced with 0s + + Raises: + ValueError if idxs belong to more than one sequence. + """ + + if subset_filter is not None: + raise NotImplementedError( + "Subset filters are not supported in SQL Dataset. " + "We encourage creating a dataset per subset." + ) + + index_slice, _ = self._get_frame_no_coalesced_ts_by_row_indices(idxs) + # alternatively, we can use `.values.tolist()`, which may be faster + # but returns a list of lists + return list(index_slice.itertuples()) + + # override + def sequence_frames_in_order( + self, seq_name: str, subset_filter: Optional[Sequence[str]] = None + ) -> Iterator[Tuple[float, int, int]]: + """ + Overrides the default DatasetBase implementation (we don’t use `_seq_to_idx`). + Returns an iterator over the frame indices in a given sequence. + We attempt to first sort by timestamp (if they are available), + then by frame number. + + Args: + seq_name: the name of the sequence. + subset_filter: subset names to filter to + + Returns: + an iterator over triplets `(timestamp, frame_no, dataset_idx)`, + where `frame_no` is the index within the sequence, and + `dataset_idx` is the index within the dataset. + `None` timestamps are replaced with 0s. + """ + # TODO: implement sort_timestamp_first? (which would matter if the orders + # of frame numbers and timestamps are different) + rows = self._index.index.get_loc(seq_name) + if isinstance(rows, slice): + assert rows.stop is not None, "Unexpected result from pandas" + rows = range(rows.start or 0, rows.stop, rows.step or 1) + else: + rows = np.where(rows)[0] + + index_slice, idx = self._get_frame_no_coalesced_ts_by_row_indices( + rows, seq_name, subset_filter + ) + index_slice["idx"] = idx + + yield from index_slice.itertuples(index=False) + + # override + def get_eval_batches(self) -> Optional[List[Any]]: + """ + This class does not support eval batches with ordinal indices. You can pass + eval_batches as a batch_sampler to a data_loader since the dataset supports + `dataset[seq_name, frame_no]` indexing. + """ + return self.eval_batches + + # override + def join(self, other_datasets: Iterable[DatasetBase]) -> None: + raise ValueError("Not supported! Preprocess the data by merging them instead.") + + # override + @property + def frame_data_type(self) -> Type[FrameData]: + return self.frame_data_builder.frame_data_type + + def is_filtered(self) -> bool: + """ + Returns `True` in case the dataset has been filtered and thus some frame + annotations stored on the disk might be missing in the dataset object. + Does not account for subsets. + + Returns: + is_filtered: `True` if the dataset has been filtered, else `False`. + """ + return ( + self.remove_empty_masks + or self.limit_to > 0 + or self.limit_sequences_to > 0 + or self.limit_sequences_per_category_to > 0 + or len(self.pick_sequences) > 0 + or len(self.exclude_sequences) > 0 + or len(self.pick_categories) > 0 + or self.n_frames_per_sequence > 0 + ) + + def _get_filtered_sequences_if_any(self) -> Optional[pd.Series]: + # maximum possible filter (if limit_sequences_per_category_to == 0): + # WHERE category IN 'self.pick_categories' + # AND sequence_name IN 'self.pick_sequences' + # AND sequence_name NOT IN 'self.exclude_sequences' + # LIMIT 'self.limit_sequence_to' + + where_conditions = [ + *self._get_category_filters(), + *self._get_pick_filters(), + *self._get_exclude_filters(), + ] + + def add_where(stmt): + return stmt.where(*where_conditions) if where_conditions else stmt + + if self.limit_sequences_per_category_to <= 0: + stmt = add_where(sa.select(SqlSequenceAnnotation.sequence_name)) + else: + subquery = sa.select( + SqlSequenceAnnotation.sequence_name, + sa.func.row_number() + .over( + order_by=sa.text("ROWID"), # NOTE: ROWID is SQLite-specific + partition_by=SqlSequenceAnnotation.category, + ) + .label("row_number"), + ) + + subquery = add_where(subquery).subquery() + stmt = sa.select(subquery.c.sequence_name).where( + subquery.c.row_number <= self.limit_sequences_per_category_to + ) + + if self.limit_sequences_to > 0: + logger.info( + f"Limiting dataset to first {self.limit_sequences_to} sequences" + ) + # NOTE: ROWID is SQLite-specific + stmt = stmt.order_by(sa.text("ROWID")).limit(self.limit_sequences_to) + + if ( + not where_conditions + and self.limit_sequences_to <= 0 + and self.limit_sequences_per_category_to <= 0 + ): + # we will not need to filter by sequences + return None + + with self._sql_engine.connect() as connection: + sequences = pd.read_sql_query(stmt, connection)["sequence_name"] + logger.info("... retained %d sequences" % len(sequences)) + + return sequences + + def _get_category_filters(self) -> List[sa.ColumnElement]: + if not self.pick_categories: + return [] + + logger.info(f"Limiting dataset to categories: {self.pick_categories}") + return [SqlSequenceAnnotation.category.in_(self.pick_categories)] + + def _get_pick_filters(self) -> List[sa.ColumnElement]: + if not self.pick_sequences: + return [] + + logger.info(f"Limiting dataset to sequences: {self.pick_sequences}") + return [SqlSequenceAnnotation.sequence_name.in_(self.pick_sequences)] + + def _get_exclude_filters(self) -> List[sa.ColumnOperators]: + if not self.exclude_sequences: + return [] + + logger.info(f"Removing sequences from the dataset: {self.exclude_sequences}") + return [SqlSequenceAnnotation.sequence_name.notin_(self.exclude_sequences)] + + def _load_subsets_from_json(self, subset_lists_path: str) -> pd.DataFrame: + assert self.subsets is not None + with open(subset_lists_path, "r") as f: + subset_to_seq_frame = json.load(f) + + seq_frame_list = sum( + ( + [(*row, subset) for row in subset_to_seq_frame[subset]] + for subset in self.subsets + ), + [], + ) + index = pd.DataFrame( + seq_frame_list, + columns=["sequence_name", "frame_number", "_image_path", "subset"], + ) + return index + + def _load_subsets_from_sql(self, subset_lists_path: str) -> pd.DataFrame: + subsets = self.subsets + assert subsets is not None + # we need a new engine since we store the subsets in a separate DB + engine = sa.create_engine(f"sqlite:///{subset_lists_path}") + table = sa.Table(_SET_LISTS_TABLE, sa.MetaData(), autoload_with=engine) + stmt = sa.select(table).where(table.c.subset.in_(subsets)) + with engine.connect() as connection: + index = pd.read_sql(stmt, connection) + + return index + + def _build_index_from_subset_lists( + self, sequences: Optional[pd.Series] + ) -> pd.DataFrame: + if not self.subset_lists_file: + raise ValueError("Requested subsets but subset_lists_file not given") + + logger.info(f"Loading subset lists from {self.subset_lists_file}.") + + subset_lists_path = self._local_path(self.subset_lists_file) + if subset_lists_path.lower().endswith(".json"): + index = self._load_subsets_from_json(subset_lists_path) + else: + index = self._load_subsets_from_sql(subset_lists_path) + index = index.set_index(["sequence_name", "frame_number"]) + logger.info(f" -> loaded {len(index)} samples of {self.subsets}.") + + if sequences is not None: + logger.info("Applying filtered sequences.") + sequence_values = index.index.get_level_values("sequence_name") + index = index.loc[sequence_values.isin(sequences)] + logger.info(f" -> retained {len(index)} samples.") + + pick_frames_criteria = [] + if self.remove_empty_masks: + logger.info("Culling samples with empty masks.") + + if len(index) > self.remove_empty_masks_poll_whole_table_threshold: + # APPROACH 1: find empty masks and drop indices. + # dev load: 17s / 15 s (3.1M / 500K) + stmt = sa.select( + self.frame_annotations_type.sequence_name, + self.frame_annotations_type.frame_number, + ).where(self.frame_annotations_type._mask_mass == 0) + with Session(self._sql_engine) as session: + to_remove = session.execute(stmt).all() + + # Pandas uses np.int64 for integer types, so we have to case + # we might want to read it to pandas DataFrame directly to avoid the loop + to_remove = [(seq, np.int64(fr)) for seq, fr in to_remove] + index.drop(to_remove, errors="ignore", inplace=True) + else: + # APPROACH 3: load index into a temp table and join with annotations + # dev load: 94 s / 23 s (3.1M / 500K) + pick_frames_criteria.append( + sa.or_( + self.frame_annotations_type._mask_mass.is_(None), + self.frame_annotations_type._mask_mass != 0, + ) + ) + + if self.pick_frames_sql_clause: + logger.info("Applying the custom SQL clause.") + pick_frames_criteria.append(sa.text(self.pick_frames_sql_clause)) + + if pick_frames_criteria: + index = self._pick_frames_by_criteria(index, pick_frames_criteria) + + logger.info(f" -> retained {len(index)} samples.") + + if self.limit_to > 0: + logger.info(f"Limiting dataset to first {self.limit_to} frames") + index = index.sort_index().iloc[: self.limit_to] + + return index.reset_index() + + def _pick_frames_by_criteria(self, index: pd.DataFrame, criteria) -> pd.DataFrame: + IndexTable = self._get_temp_index_table_instance() + with self._sql_engine.connect() as connection: + IndexTable.create(connection) + # we don’t let pandas’s `to_sql` create the table automatically as + # the table would be permanent, so we create it and append with pandas + n_rows = index.to_sql(IndexTable.name, connection, if_exists="append") + assert n_rows == len(index) + sa_type = self.frame_annotations_type + stmt = ( + sa.select(IndexTable) + .select_from( + IndexTable.join( + self.frame_annotations_type, + sa.and_( + sa_type.sequence_name == IndexTable.c.sequence_name, + sa_type.frame_number == IndexTable.c.frame_number, + ), + ) + ) + .where(*criteria) + ) + return pd.read_sql_query(stmt, connection).set_index( + ["sequence_name", "frame_number"] + ) + + def _build_index_from_db(self, sequences: Optional[pd.Series]): + logger.info("Loading sequcence-frame index from the database") + stmt = sa.select( + self.frame_annotations_type.sequence_name, + self.frame_annotations_type.frame_number, + self.frame_annotations_type._image_path, + sa.null().label("subset"), + ) + where_conditions = [] + if sequences is not None: + logger.info(" applying filtered sequences") + where_conditions.append( + self.frame_annotations_type.sequence_name.in_(sequences.tolist()) + ) + + if self.remove_empty_masks: + logger.info(" excluding samples with empty masks") + where_conditions.append( + sa.or_( + self.frame_annotations_type._mask_mass.is_(None), + self.frame_annotations_type._mask_mass != 0, + ) + ) + + if self.pick_frames_sql_clause: + logger.info(" applying custom SQL clause") + where_conditions.append(sa.text(self.pick_frames_sql_clause)) + + if where_conditions: + stmt = stmt.where(*where_conditions) + + if self.limit_to > 0: + logger.info(f"Limiting dataset to first {self.limit_to} frames") + stmt = stmt.order_by( + self.frame_annotations_type.sequence_name, + self.frame_annotations_type.frame_number, + ).limit(self.limit_to) + + with self._sql_engine.connect() as connection: + index = pd.read_sql_query(stmt, connection) + + logger.info(f" -> loaded {len(index)} samples.") + return index + + def _sort_index_(self, index): + logger.info("Sorting the index by sequence and frame number.") + index.sort_values(["sequence_name", "frame_number"], inplace=True) + logger.info(" -> Done.") + + def _load_filter_eval_batches(self): + assert self.eval_batches_file + logger.info(f"Loading eval batches from {self.eval_batches_file}") + + if not os.path.isfile(self.eval_batches_file): + # The batch indices file does not exist. + # Most probably the user has not specified the root folder. + raise ValueError( + f"Looking for dataset json file in {self.eval_batches_file}. " + + "Please specify a correct dataset_root folder." + ) + + with open(self.eval_batches_file, "r") as f: + eval_batches = json.load(f) + + # limit the dataset to sequences to allow multiple evaluations in one file + pick_sequences = set(self.pick_sequences) + if self.pick_categories: + cat_to_seq = self.category_to_sequence_names() + pick_sequences.update( + seq for cat in self.pick_categories for seq in cat_to_seq[cat] + ) + + if pick_sequences: + old_len = len(eval_batches) + eval_batches = [b for b in eval_batches if b[0][0] in pick_sequences] + logger.warn( + f"Picked eval batches by sequence/cat: {old_len} -> {len(eval_batches)}" + ) + + if self.exclude_sequences: + old_len = len(eval_batches) + exclude_sequences = set(self.exclude_sequences) + eval_batches = [b for b in eval_batches if b[0][0] not in exclude_sequences] + logger.warn( + f"Excluded eval batches by sequence: {old_len} -> {len(eval_batches)}" + ) + + return eval_batches + + def _stratified_sample_index(self, index): + # NOTE this stratified sampling can be done more efficiently in + # the no-subset case above if it is added to the SQL query. + # We keep this generic implementation since no-subset case is uncommon + index = index.groupby("sequence_name", group_keys=False).apply( + lambda seq_frames: seq_frames.sample( + min(len(seq_frames), self.n_frames_per_sequence), + random_state=( + _seq_name_to_seed(seq_frames.iloc[0]["sequence_name"]) + self.seed + ), + ) + ) + logger.info(f" -> retained {len(index)} samples aster stratified sampling.") + return index + + def _get_frame_type(self, entry: SqlFrameAnnotation) -> Optional[str]: + return self._index.loc[(entry.sequence_name, entry.frame_number), "subset"] + + def _get_frame_no_coalesced_ts_by_row_indices( + self, + idxs: Sequence[int], + seq_name: Optional[str] = None, + subset_filter: Union[Sequence[str], str, None] = None, + ) -> Tuple[pd.DataFrame, Sequence[int]]: + """ + Loads timestamps for given index rows belonging to the same sequence. + If seq_name is known, it speeds up the computation. + Raises ValueError if `idxs` do not all belong to a single sequences . + """ + index_slice = self._index.iloc[idxs] + if subset_filter is not None: + if isinstance(subset_filter, str): + subset_filter = [subset_filter] + indicator = index_slice["subset"].isin(subset_filter) + index_slice = index_slice.loc[indicator] + idxs = [i for i, isin in zip(idxs, indicator) if isin] + + frames = index_slice.index.get_level_values("frame_number").tolist() + if seq_name is None: + seq_name_list = index_slice.index.get_level_values("sequence_name").tolist() + seq_name_set = set(seq_name_list) + if len(seq_name_set) > 1: + raise ValueError("Given indices belong to more than one sequence.") + elif len(seq_name_set) == 1: + seq_name = seq_name_list[0] + + coalesced_ts = sa.sql.functions.coalesce( + self.frame_annotations_type.frame_timestamp, 0 + ) + stmt = sa.select( + coalesced_ts.label("frame_timestamp"), + self.frame_annotations_type.frame_number, + ).where( + self.frame_annotations_type.sequence_name == seq_name, + self.frame_annotations_type.frame_number.in_(frames), + ) + + with self._sql_engine.connect() as connection: + frame_no_ts = pd.read_sql_query(stmt, connection) + + if len(frame_no_ts) != len(index_slice): + raise ValueError( + "Not all indices are found in the database; " + "do they belong to more than one sequence?" + ) + + return frame_no_ts, idxs + + def _local_path(self, path: str) -> str: + if self.path_manager is None: + return path + return self.path_manager.get_local_path(path) + + def _get_temp_index_table_instance(self, table_name: str = "__index"): + CachedTable = self.frame_annotations_type.metadata.tables.get(table_name) + if CachedTable is not None: # table definition is not idempotent + return CachedTable + + return sa.Table( + table_name, + self.frame_annotations_type.metadata, + sa.Column("sequence_name", sa.String, primary_key=True), + sa.Column("frame_number", sa.Integer, primary_key=True), + sa.Column("_image_path", sa.String), + sa.Column("subset", sa.String), + prefixes=["TEMP"], # NOTE SQLite specific! + ) + + +def _seq_name_to_seed(seq_name) -> int: + """Generates numbers in [0, 2 ** 28)""" + return int(hashlib.sha1(seq_name.encode("utf-8")).hexdigest()[:7], 16) + + +def _safe_as_tensor(data, dtype): + return torch.tensor(data, dtype=dtype) if data is not None else None diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/sql_dataset_provider.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/sql_dataset_provider.py new file mode 100644 index 0000000000000000000000000000000000000000..ab161e8d73b3c1e7f0f00195fd6fdf94bf9b7af1 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/sql_dataset_provider.py @@ -0,0 +1,424 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + + +import logging +import os +from typing import List, Optional, Tuple, Type + +import numpy as np + +from omegaconf import DictConfig, OmegaConf + +from pytorch3d.implicitron.dataset.dataset_map_provider import ( + DatasetMap, + DatasetMapProviderBase, + PathManagerFactory, +) +from pytorch3d.implicitron.tools.config import ( + expand_args_fields, + registry, + run_auto_creation, +) + +from .sql_dataset import SqlIndexDataset + + +_CO3D_SQL_DATASET_ROOT: str = os.getenv("CO3D_SQL_DATASET_ROOT", "") + +# _NEED_CONTROL is a list of those elements of SqlIndexDataset which +# are not directly specified for it in the config but come from the +# DatasetMapProvider. +_NEED_CONTROL: Tuple[str, ...] = ( + "path_manager", + "subsets", + "sqlite_metadata_file", + "subset_lists_file", +) + +logger = logging.getLogger(__name__) + + +@registry.register +class SqlIndexDatasetMapProvider(DatasetMapProviderBase): # pyre-ignore [13] + """ + Generates the training, validation, and testing dataset objects for + a dataset laid out on disk like SQL-CO3D, with annotations in an SQLite data base. + + The dataset is organized in the filesystem as follows:: + + self.dataset_root + ├── + │ ├── + │ │ ├── depth_masks + │ │ ├── depths + │ │ ├── images + │ │ ├── masks + │ │ └── pointcloud.ply + │ ├── + │ │ ├── depth_masks + │ │ ├── depths + │ │ ├── images + │ │ ├── masks + │ │ └── pointcloud.ply + │ ├── ... + │ ├── + │ ├── set_lists + │ ├── .json + │ ├── .json + │ ├── ... + │ ├── .json + │ ├── eval_batches + │ │ ├── .json + │ │ ├── .json + │ │ ├── ... + │ │ ├── .json + │ ├── frame_annotations.jgz + │ ├── sequence_annotations.jgz + ├── + ├── ... + ├── + ├── set_lists + ├── .sqlite + ├── .sqlite + ├── ... + ├── .sqlite + ├── eval_batches + │ ├── .json + │ ├── .json + │ ├── ... + │ ├── .json + + The dataset contains sequences named `` that may be partitioned by + directories such as `` e.g. representing categories but they + can also be stored in a flat structure. Each sequence folder contains the list of + sequence images, depth maps, foreground masks, and valid-depth masks + `images`, `depths`, `masks`, and `depth_masks` respectively. Furthermore, + `set_lists/` dirtectories (with partitions or global) store json or sqlite files + `.`, each describing a certain sequence subset. + These subset path conventions are not hard-coded and arbitrary relative path can be + specified by setting `self.subset_lists_path` to the relative path w.r.t. + dataset root. + + Each `.json` file contains the following dictionary:: + + { + "train": [ + (sequence_name: str, frame_number: int, image_path: str), + ... + ], + "val": [ + (sequence_name: str, frame_number: int, image_path: str), + ... + ], + "test": [ + (sequence_name: str, frame_number: int, image_path: str), + ... + ], + ] + + defining the list of frames (identified with their `sequence_name` and + `frame_number`) in the "train", "val", and "test" subsets of the dataset. In case of + SQLite format, `.sqlite` contains a table with the header:: + + | sequence_name | frame_number | image_path | subset | + + Note that `frame_number` can be obtained only from the metadata and + does not necesarrily correspond to the numeric suffix of the corresponding image + file name (e.g. a file `//images/frame00005.jpg` can + have its frame number set to `20`, not 5). + + Each `.json` file contains a list of evaluation examples + in the following form:: + + [ + [ # batch 1 + (sequence_name: str, frame_number: int, image_path: str), + ... + ], + [ # batch 2 + (sequence_name: str, frame_number: int, image_path: str), + ... + ], + ] + + Note that the evaluation examples always come from the `"test"` subset of the dataset. + (test frames can repeat across batches). The batches can contain single element, + which is typical in case of regular radiance field fitting. + + Args: + subset_lists_path: The relative path to the dataset subset definition. + For CO3D, these include e.g. "skateboard/set_lists/set_lists_manyview_dev_0.json". + By default (None), dataset is not partitioned to subsets (in that case, setting + `ignore_subsets` will speed up construction) + dataset_root: The root folder of the dataset. + metadata_basename: name of the SQL metadata file in dataset_root; + not expected to be changed by users + test_on_train: Construct validation and test datasets from + the training subset; note that in practice, in this + case all subset dataset objects will be same + only_test_set: Load only the test set. Incompatible with `test_on_train`. + ignore_subsets: Don’t filter by subsets in the dataset; note that in this + case all subset datasets will be same + eval_batch_num_training_frames: Add a certain number of training frames to each + eval batch. Useful for evaluating models that require + source views as input (e.g. NeRF-WCE / PixelNeRF). + dataset_args: Specifies additional arguments to the + JsonIndexDataset constructor call. + path_manager_factory: (Optional) An object that generates an instance of + PathManager that can translate provided file paths. + path_manager_factory_class_type: The class type of `path_manager_factory`. + """ + + category: Optional[str] = None + subset_list_name: Optional[str] = None # TODO: docs + # OR + subset_lists_path: Optional[str] = None + eval_batches_path: Optional[str] = None + + dataset_root: str = _CO3D_SQL_DATASET_ROOT + metadata_basename: str = "metadata.sqlite" + + test_on_train: bool = False + only_test_set: bool = False + ignore_subsets: bool = False + train_subsets: Tuple[str, ...] = ("train",) + val_subsets: Tuple[str, ...] = ("val",) + test_subsets: Tuple[str, ...] = ("test",) + + eval_batch_num_training_frames: int = 0 + + # this is a mould that is never constructed, used to build self._dataset_map values + dataset_class_type: str = "SqlIndexDataset" + dataset: SqlIndexDataset + + path_manager_factory: PathManagerFactory + path_manager_factory_class_type: str = "PathManagerFactory" + + def __post_init__(self): + super().__init__() + run_auto_creation(self) + + if self.only_test_set and self.test_on_train: + raise ValueError("Cannot have only_test_set and test_on_train") + + if self.ignore_subsets and not self.only_test_set: + self.test_on_train = True # no point in loading same data 3 times + + path_manager = self.path_manager_factory.get() + + sqlite_metadata_file = os.path.join(self.dataset_root, self.metadata_basename) + sqlite_metadata_file = _local_path(path_manager, sqlite_metadata_file) + + if not os.path.isfile(sqlite_metadata_file): + # The sqlite_metadata_file does not exist. + # Most probably the user has not specified the root folder. + raise ValueError( + f"Looking for frame annotations in {sqlite_metadata_file}." + + " Please specify a correct dataset_root folder." + + " Note: By default the root folder is taken from the" + + " CO3D_SQL_DATASET_ROOT environment variable." + ) + + if self.subset_lists_path and self.subset_list_name: + raise ValueError( + "subset_lists_path and subset_list_name cannot be both set" + ) + + subset_lists_file = self._get_lists_file("set_lists") + + # setup the common dataset arguments + common_dataset_kwargs = { + **getattr(self, f"dataset_{self.dataset_class_type}_args"), + "sqlite_metadata_file": sqlite_metadata_file, + "dataset_root": self.dataset_root, + "subset_lists_file": subset_lists_file, + "path_manager": path_manager, + } + + if self.category: + logger.info(f"Forcing category filter in the datasets to {self.category}") + common_dataset_kwargs["pick_categories"] = self.category.split(",") + + # get the used dataset type + dataset_type: Type[SqlIndexDataset] = registry.get( + SqlIndexDataset, self.dataset_class_type + ) + expand_args_fields(dataset_type) + + if subset_lists_file is not None and not os.path.isfile(subset_lists_file): + available_subsets = self._get_available_subsets( + OmegaConf.to_object(common_dataset_kwargs["pick_categories"]) + ) + msg = f"Cannot find subset list file {self.subset_lists_path}." + if available_subsets: + msg += f" Some of the available subsets: {str(available_subsets)}." + raise ValueError(msg) + + train_dataset = None + val_dataset = None + if not self.only_test_set: + # load the training set + logger.debug("Constructing train dataset.") + train_dataset = dataset_type( + **common_dataset_kwargs, subsets=self._get_subsets(self.train_subsets) + ) + logger.info(f"Train dataset: {str(train_dataset)}") + + if self.test_on_train: + assert train_dataset is not None + val_dataset = test_dataset = train_dataset + else: + # load the val and test sets + if not self.only_test_set: + # NOTE: this is always loaded in JsonProviderV2 + logger.debug("Extracting val dataset.") + val_dataset = dataset_type( + **common_dataset_kwargs, subsets=self._get_subsets(self.val_subsets) + ) + logger.info(f"Val dataset: {str(val_dataset)}") + + logger.debug("Extracting test dataset.") + eval_batches_file = self._get_lists_file("eval_batches") + del common_dataset_kwargs["eval_batches_file"] + test_dataset = dataset_type( + **common_dataset_kwargs, + subsets=self._get_subsets(self.test_subsets, True), + eval_batches_file=eval_batches_file, + ) + logger.info(f"Test dataset: {str(test_dataset)}") + + if ( + eval_batches_file is not None + and self.eval_batch_num_training_frames > 0 + ): + self._extend_eval_batches(test_dataset) + + self._dataset_map = DatasetMap( + train=train_dataset, val=val_dataset, test=test_dataset + ) + + def _get_subsets(self, subsets, is_eval: bool = False): + if self.ignore_subsets: + return None + + if is_eval and self.eval_batch_num_training_frames > 0: + # we will need to have training frames for extended batches + return list(subsets) + list(self.train_subsets) + + return subsets + + def _extend_eval_batches(self, test_dataset: SqlIndexDataset) -> None: + rng = np.random.default_rng(seed=0) + eval_batches = test_dataset.get_eval_batches() + if eval_batches is None: + raise ValueError("Eval batches were not loaded!") + + for batch in eval_batches: + sequence = batch[0][0] + seq_frames = list( + test_dataset.sequence_frames_in_order(sequence, self.train_subsets) + ) + idx_to_add = rng.permutation(len(seq_frames))[ + : self.eval_batch_num_training_frames + ] + batch.extend((sequence, seq_frames[a][1]) for a in idx_to_add) + + @classmethod + def dataset_tweak_args(cls, type, args: DictConfig) -> None: + """ + Called by get_default_args. + Certain fields are not exposed on each dataset class + but rather are controlled by this provider class. + """ + for key in _NEED_CONTROL: + del args[key] + + def create_dataset(self): + # No `dataset` member of this class is created. + # The dataset(s) live in `self.get_dataset_map`. + pass + + def get_dataset_map(self) -> DatasetMap: + return self._dataset_map # pyre-ignore [16] + + def _get_available_subsets(self, categories: List[str]): + """ + Get the available subset names for a given category folder (if given) inside + a root dataset folder `dataset_root`. + """ + path_manager = self.path_manager_factory.get() + + subsets: List[str] = [] + for prefix in [""] + categories: + set_list_dir = os.path.join(self.dataset_root, prefix, "set_lists") + if not ( + (path_manager is not None) and path_manager.isdir(set_list_dir) + ) and not os.path.isdir(set_list_dir): + continue + + set_list_files = (os.listdir if path_manager is None else path_manager.ls)( + set_list_dir + ) + subsets.extend(os.path.join(prefix, "set_lists", f) for f in set_list_files) + + return subsets + + def _get_lists_file(self, flavor: str) -> Optional[str]: + if flavor == "eval_batches": + subset_lists_path = self.eval_batches_path + else: + subset_lists_path = self.subset_lists_path + + if not subset_lists_path and not self.subset_list_name: + return None + + category_elem = "" + if self.category and "," not in self.category: + # if multiple categories are given, looking for global set lists + category_elem = self.category + + subset_lists_path = subset_lists_path or ( + os.path.join( + category_elem, f"{flavor}", f"{flavor}_{self.subset_list_name}" + ) + ) + + assert subset_lists_path + path_manager = self.path_manager_factory.get() + # try absolute path first + subset_lists_file = _get_local_path_check_extensions( + subset_lists_path, path_manager + ) + if subset_lists_file: + return subset_lists_file + + full_path = os.path.join(self.dataset_root, subset_lists_path) + subset_lists_file = _get_local_path_check_extensions(full_path, path_manager) + + if not subset_lists_file: + raise FileNotFoundError( + f"Subset lists path given but not found: {full_path}" + ) + + return subset_lists_file + + +def _get_local_path_check_extensions( + path, path_manager, extensions=("", ".sqlite", ".json") +) -> Optional[str]: + for ext in extensions: + local = _local_path(path_manager, path + ext) + if os.path.isfile(local): + return local + + return None + + +def _local_path(path_manager, path: str) -> str: + if path_manager is None: + return path + return path_manager.get_local_path(path) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/train_eval_data_loader_provider.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/train_eval_data_loader_provider.py new file mode 100644 index 0000000000000000000000000000000000000000..c7c0c39a4c0ea852fae3e634ccc7a6a40e2ae766 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/train_eval_data_loader_provider.py @@ -0,0 +1,191 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import logging +from typing import Any, Dict, Optional, Tuple + +from pytorch3d.implicitron.dataset.data_loader_map_provider import ( + DataLoaderMap, + SceneBatchSampler, + SequenceDataLoaderMapProvider, +) +from pytorch3d.implicitron.dataset.dataset_base import DatasetBase +from pytorch3d.implicitron.dataset.dataset_map_provider import DatasetMap +from pytorch3d.implicitron.dataset.frame_data import FrameData +from pytorch3d.implicitron.tools.config import registry, run_auto_creation + +from torch.utils.data import DataLoader + +logger = logging.getLogger(__name__) + + +# TODO: we can merge it with SequenceDataLoaderMapProvider in PyTorch3D +# and support both eval_batches protocols +@registry.register +class TrainEvalDataLoaderMapProvider(SequenceDataLoaderMapProvider): + """ + Implementation of DataLoaderMapProviderBase that may use internal eval batches for + the test dataset. In particular, if `eval_batches_relpath` is set, it loads + eval batches from that json file, otherwise test set is treated in the same way as + train and val, i.e. the parameters `dataset_length_test` and `test_conditioning_type` + are respected. + + If conditioning is not required, then the batch size should + be set as 1, and most of the fields do not matter. + + If conditioning is required, each batch will contain one main + frame first to predict and the, rest of the elements are for + conditioning. + + If images_per_seq_options is left empty, the conditioning + frames are picked according to the conditioning type given. + This does not have regard to the order of frames in a + scene, or which frames belong to what scene. + + If images_per_seq_options is given, then the conditioning types + must be SAME and the remaining fields are used. + + Members: + batch_size: The size of the batch of the data loader. + num_workers: Number of data-loading threads in each data loader. + dataset_length_train: The number of batches in a training epoch. Or 0 to mean + an epoch is the length of the training set. + dataset_length_val: The number of batches in a validation epoch. Or 0 to mean + an epoch is the length of the validation set. + dataset_length_test: used if test_dataset.eval_batches is NOT set. The number of + batches in a testing epoch. Or 0 to mean an epoch is the length of the test + set. + images_per_seq_options: Possible numbers of frames sampled per sequence in a batch. + If a conditioning_type is KNOWN or TRAIN, then this must be left at its initial + value. Empty (the default) means that we are not careful about which frames + come from which scene. + sample_consecutive_frames: if True, will sample a contiguous interval of frames + in the sequence. It first sorts the frames by timestimps when available, + otherwise by frame numbers, finds the connected segments within the sequence + of sufficient length, then samples a random pivot element among them and + ideally uses it as a middle of the temporal window, shifting the borders + where necessary. This strategy mitigates the bias against shorter segments + and their boundaries. + consecutive_frames_max_gap: if a number > 0, then used to define the maximum + difference in frame_number of neighbouring frames when forming connected + segments; if both this and consecutive_frames_max_gap_seconds are 0s, + the whole sequence is considered a segment regardless of frame numbers. + consecutive_frames_max_gap_seconds: if a number > 0.0, then used to define the + maximum difference in frame_timestamp of neighbouring frames when forming + connected segments; if both this and consecutive_frames_max_gap are 0s, + the whole sequence is considered a segment regardless of frame timestamps. + """ + + batch_size: int = 1 + num_workers: int = 0 + + dataset_length_train: int = 0 + dataset_length_val: int = 0 + dataset_length_test: int = 0 + + images_per_seq_options: Tuple[int, ...] = () + sample_consecutive_frames: bool = False + consecutive_frames_max_gap: int = 0 + consecutive_frames_max_gap_seconds: float = 0.1 + + def __post_init__(self): + run_auto_creation(self) + + def get_data_loader_map(self, datasets: DatasetMap) -> DataLoaderMap: + """ + Returns a collection of data loaders for a given collection of datasets. + """ + train = self._make_generic_data_loader( + datasets.train, + self.dataset_length_train, + datasets.train, + ) + + val = self._make_generic_data_loader( + datasets.val, + self.dataset_length_val, + datasets.train, + ) + + if datasets.test is not None and datasets.test.get_eval_batches() is not None: + test = self._make_eval_data_loader(datasets.test) + else: + test = self._make_generic_data_loader( + datasets.test, + self.dataset_length_test, + datasets.train, + ) + + return DataLoaderMap(train=train, val=val, test=test) + + def _make_eval_data_loader( + self, + dataset: Optional[DatasetBase], + ) -> Optional[DataLoader[FrameData]]: + if dataset is None: + return None + + return DataLoader( + dataset, + batch_sampler=dataset.get_eval_batches(), + **self._get_data_loader_common_kwargs(dataset), + ) + + def _make_generic_data_loader( + self, + dataset: Optional[DatasetBase], + num_batches: int, + train_dataset: Optional[DatasetBase], + ) -> Optional[DataLoader[FrameData]]: + """ + Returns the dataloader for a dataset. + + Args: + dataset: the dataset + num_batches: possible ceiling on number of batches per epoch + train_dataset: the training dataset, used if conditioning_type==TRAIN + conditioning_type: source for padding of batches + """ + if dataset is None: + return None + + data_loader_kwargs = self._get_data_loader_common_kwargs(dataset) + + if len(self.images_per_seq_options) > 0: + # this is a typical few-view setup + # conditioning comes from the same subset since subsets are split by seqs + batch_sampler = SceneBatchSampler( + dataset, + self.batch_size, + num_batches=len(dataset) if num_batches <= 0 else num_batches, + images_per_seq_options=self.images_per_seq_options, + sample_consecutive_frames=self.sample_consecutive_frames, + consecutive_frames_max_gap=self.consecutive_frames_max_gap, + consecutive_frames_max_gap_seconds=self.consecutive_frames_max_gap_seconds, + ) + return DataLoader( + dataset, + batch_sampler=batch_sampler, + **data_loader_kwargs, + ) + + if self.batch_size == 1: + # this is a typical many-view setup (without conditioning) + return self._simple_loader(dataset, num_batches, data_loader_kwargs) + + # edge case: conditioning on train subset, typical for Nerformer-like many-view + # there is only one sequence in all datasets, so we condition on another subset + return self._train_loader( + dataset, train_dataset, num_batches, data_loader_kwargs + ) + + def _get_data_loader_common_kwargs(self, dataset: DatasetBase) -> Dict[str, Any]: + return { + "num_workers": self.num_workers, + "collate_fn": dataset.frame_data_type.collate, + } diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/types.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/types.py new file mode 100644 index 0000000000000000000000000000000000000000..faf8c790a7d2e46eb7a585b745b711f830ea172c --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/types.py @@ -0,0 +1,357 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +import dataclasses +import gzip +import json +from dataclasses import dataclass, Field, MISSING +from typing import ( + Any, + cast, + Dict, + get_args, + get_origin, + IO, + Optional, + Tuple, + Type, + TypeVar, + Union, +) + +import numpy as np + + +_X = TypeVar("_X") + +TF3 = Tuple[float, float, float] + + +@dataclass +class ImageAnnotation: + # path to jpg file, relative w.r.t. dataset_root + path: str + # H x W + size: Tuple[int, int] # TODO: rename size_hw? + + +@dataclass +class DepthAnnotation: + # path to png file, relative w.r.t. dataset_root, storing `depth / scale_adjustment` + path: str + # a factor to convert png values to actual depth: `depth = png * scale_adjustment` + scale_adjustment: float + # path to png file, relative w.r.t. dataset_root, storing binary `depth` mask + mask_path: Optional[str] + + +@dataclass +class MaskAnnotation: + # path to png file storing (Prob(fg | pixel) * 255) + path: str + # (soft) number of pixels in the mask; sum(Prob(fg | pixel)) + mass: Optional[float] = None + # tight bounding box around the foreground mask + bounding_box_xywh: Optional[Tuple[float, float, float, float]] = None + + +@dataclass +class ViewpointAnnotation: + # In right-multiply (PyTorch3D) format. X_cam = X_world @ R + T + R: Tuple[TF3, TF3, TF3] + T: TF3 + + focal_length: Tuple[float, float] + principal_point: Tuple[float, float] + + intrinsics_format: str = "ndc_norm_image_bounds" + # Defines the co-ordinate system where focal_length and principal_point live. + # Possible values: ndc_isotropic | ndc_norm_image_bounds (default) + # ndc_norm_image_bounds: legacy PyTorch3D NDC format, where image boundaries + # correspond to [-1, 1] x [-1, 1], and the scale along x and y may differ + # ndc_isotropic: PyTorch3D 0.5+ NDC convention where the shorter side has + # the range [-1, 1], and the longer one has the range [-s, s]; s >= 1, + # where s is the aspect ratio. The scale is same along x and y. + + +@dataclass +class FrameAnnotation: + """A dataclass used to load annotations from json.""" + + # can be used to join with `SequenceAnnotation` + sequence_name: str + # 0-based, continuous frame number within sequence + frame_number: int + # timestamp in seconds from the video start + frame_timestamp: float + + image: ImageAnnotation + depth: Optional[DepthAnnotation] = None + mask: Optional[MaskAnnotation] = None + viewpoint: Optional[ViewpointAnnotation] = None + meta: Optional[Dict[str, Any]] = None + + +@dataclass +class PointCloudAnnotation: + # path to ply file with points only, relative w.r.t. dataset_root + path: str + # the bigger the better + quality_score: float + n_points: Optional[int] + + +@dataclass +class VideoAnnotation: + # path to the original video file, relative w.r.t. dataset_root + path: str + # length of the video in seconds + length: float + + +@dataclass +class SequenceAnnotation: + sequence_name: str + category: str + video: Optional[VideoAnnotation] = None + point_cloud: Optional[PointCloudAnnotation] = None + # the bigger the better + viewpoint_quality_score: Optional[float] = None + + +def dump_dataclass(obj: Any, f: IO, binary: bool = False) -> None: + """ + Args: + f: Either a path to a file, or a file opened for writing. + obj: A @dataclass or collection hierarchy including dataclasses. + binary: Set to True if `f` is a file handle, else False. + """ + if binary: + f.write(json.dumps(_asdict_rec(obj)).encode("utf8")) + else: + json.dump(_asdict_rec(obj), f) + + +def load_dataclass(f: IO, cls: Type[_X], binary: bool = False) -> _X: + """ + Loads to a @dataclass or collection hierarchy including dataclasses + from a json recursively. + Call it like load_dataclass(f, typing.List[FrameAnnotationAnnotation]). + raises KeyError if json has keys not mapping to the dataclass fields. + + Args: + f: Either a path to a file, or a file opened for writing. + cls: The class of the loaded dataclass. + binary: Set to True if `f` is a file handle, else False. + """ + if binary: + asdict = json.loads(f.read().decode("utf8")) + else: + asdict = json.load(f) + + if isinstance(asdict, list): + # in the list case, run a faster "vectorized" version + cls = get_args(cls)[0] + res = list(_dataclass_list_from_dict_list(asdict, cls)) + else: + res = _dataclass_from_dict(asdict, cls) + + return res + + +def _dataclass_list_from_dict_list(dlist, typeannot): + """ + Vectorised version of `_dataclass_from_dict`. + The output should be equivalent to + `[_dataclass_from_dict(d, typeannot) for d in dlist]`. + + Args: + dlist: list of objects to convert. + typeannot: type of each of those objects. + Returns: + iterator or list over converted objects of the same length as `dlist`. + + Raises: + ValueError: it assumes the objects have None's in consistent places across + objects, otherwise it would ignore some values. This generally holds for + auto-generated annotations, but otherwise use `_dataclass_from_dict`. + """ + + cls = get_origin(typeannot) or typeannot + + if typeannot is Any: + return dlist + if all(obj is None for obj in dlist): # 1st recursion base: all None nodes + return dlist + if any(obj is None for obj in dlist): + # filter out Nones and recurse on the resulting list + idx_notnone = [(i, obj) for i, obj in enumerate(dlist) if obj is not None] + idx, notnone = zip(*idx_notnone) + converted = _dataclass_list_from_dict_list(notnone, typeannot) + res = [None] * len(dlist) + for i, obj in zip(idx, converted): + res[i] = obj + return res + + is_optional, contained_type = _resolve_optional(typeannot) + if is_optional: + return _dataclass_list_from_dict_list(dlist, contained_type) + + # otherwise, we dispatch by the type of the provided annotation to convert to + if issubclass(cls, tuple) and hasattr(cls, "_fields"): # namedtuple + # For namedtuple, call the function recursively on the lists of corresponding keys + types = cls.__annotations__.values() + dlist_T = zip(*dlist) + res_T = [ + _dataclass_list_from_dict_list(key_list, tp) + for key_list, tp in zip(dlist_T, types) + ] + return [cls(*converted_as_tuple) for converted_as_tuple in zip(*res_T)] + elif issubclass(cls, (list, tuple)): + # For list/tuple, call the function recursively on the lists of corresponding positions + types = get_args(typeannot) + if len(types) == 1: # probably List; replicate for all items + types = types * len(dlist[0]) + dlist_T = zip(*dlist) + res_T = ( + _dataclass_list_from_dict_list(pos_list, tp) + for pos_list, tp in zip(dlist_T, types) + ) + if issubclass(cls, tuple): + return list(zip(*res_T)) + else: + return [cls(converted_as_tuple) for converted_as_tuple in zip(*res_T)] + elif issubclass(cls, dict): + # For the dictionary, call the function recursively on concatenated keys and vertices + key_t, val_t = get_args(typeannot) + all_keys_res = _dataclass_list_from_dict_list( + [k for obj in dlist for k in obj.keys()], key_t + ) + all_vals_res = _dataclass_list_from_dict_list( + [k for obj in dlist for k in obj.values()], val_t + ) + indices = np.cumsum([len(obj) for obj in dlist]) + assert indices[-1] == len(all_keys_res) + + keys = np.split(list(all_keys_res), indices[:-1]) + all_vals_res_iter = iter(all_vals_res) + return [cls(zip(k, all_vals_res_iter)) for k in keys] + elif not dataclasses.is_dataclass(typeannot): + return dlist + + # dataclass node: 2nd recursion base; call the function recursively on the lists + # of the corresponding fields + assert dataclasses.is_dataclass(cls) + fieldtypes = { + f.name: (_unwrap_type(f.type), _get_dataclass_field_default(f)) + for f in dataclasses.fields(typeannot) + } + + # NOTE the default object is shared here + key_lists = ( + _dataclass_list_from_dict_list([obj.get(k, default) for obj in dlist], type_) + for k, (type_, default) in fieldtypes.items() + ) + transposed = zip(*key_lists) + return [cls(*vals_as_tuple) for vals_as_tuple in transposed] + + +def _dataclass_from_dict(d, typeannot): + if d is None or typeannot is Any: + return d + is_optional, contained_type = _resolve_optional(typeannot) + if is_optional: + # an Optional not set to None, just use the contents of the Optional. + return _dataclass_from_dict(d, contained_type) + + cls = get_origin(typeannot) or typeannot + if issubclass(cls, tuple) and hasattr(cls, "_fields"): # namedtuple + types = cls.__annotations__.values() + return cls(*[_dataclass_from_dict(v, tp) for v, tp in zip(d, types)]) + elif issubclass(cls, (list, tuple)): + types = get_args(typeannot) + if len(types) == 1: # probably List; replicate for all items + types = types * len(d) + return cls(_dataclass_from_dict(v, tp) for v, tp in zip(d, types)) + elif issubclass(cls, dict): + key_t, val_t = get_args(typeannot) + return cls( + (_dataclass_from_dict(k, key_t), _dataclass_from_dict(v, val_t)) + for k, v in d.items() + ) + elif not dataclasses.is_dataclass(typeannot): + return d + + assert dataclasses.is_dataclass(cls) + fieldtypes = {f.name: _unwrap_type(f.type) for f in dataclasses.fields(typeannot)} + return cls(**{k: _dataclass_from_dict(v, fieldtypes[k]) for k, v in d.items()}) + + +def _unwrap_type(tp): + # strips Optional wrapper, if any + if get_origin(tp) is Union: + args = get_args(tp) + if len(args) == 2 and any(a is type(None) for a in args): # noqa: E721 + # this is typing.Optional + return args[0] if args[1] is type(None) else args[1] # noqa: E721 + return tp + + +def _get_dataclass_field_default(field: Field) -> Any: + if field.default_factory is not MISSING: + # pyre-fixme[29]: `Union[dataclasses._MISSING_TYPE, + # dataclasses._DefaultFactory[typing.Any]]` is not a function. + return field.default_factory() + elif field.default is not MISSING: + return field.default + else: + return None + + +def _asdict_rec(obj): + return dataclasses._asdict_inner(obj, dict) + + +def dump_dataclass_jgzip(outfile: str, obj: Any) -> None: + """ + Dumps obj to a gzipped json outfile. + + Args: + obj: A @dataclass or collection hiererchy including dataclasses. + outfile: The path to the output file. + """ + with gzip.GzipFile(outfile, "wb") as f: + dump_dataclass(obj, cast(IO, f), binary=True) + + +def load_dataclass_jgzip(outfile, cls): + """ + Loads a dataclass from a gzipped json outfile. + + Args: + outfile: The path to the loaded file. + cls: The type annotation of the loaded dataclass. + + Returns: + loaded_dataclass: The loaded dataclass. + """ + with gzip.GzipFile(outfile, "rb") as f: + return load_dataclass(cast(IO, f), cls, binary=True) + + +def _resolve_optional(type_: Any) -> Tuple[bool, Any]: + """Check whether `type_` is equivalent to `typing.Optional[T]` for some T.""" + if get_origin(type_) is Union: + args = get_args(type_) + if len(args) == 2 and args[1] == type(None): # noqa E721 + return True, args[0] + if type_ is Any: + return True, Any + + return False, type_ diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/utils.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b8c09518244c303e45a87e35790b8d4e3d6d3019 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/utils.py @@ -0,0 +1,382 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +import functools +import warnings +from pathlib import Path +from typing import List, Optional, Tuple, TypeVar, Union + +import numpy as np +import torch +from PIL import Image + +from pytorch3d.io import IO +from pytorch3d.renderer.cameras import PerspectiveCameras +from pytorch3d.structures.pointclouds import Pointclouds + +DATASET_TYPE_TRAIN = "train" +DATASET_TYPE_TEST = "test" +DATASET_TYPE_KNOWN = "known" +DATASET_TYPE_UNKNOWN = "unseen" + + +class GenericWorkaround: + """ + OmegaConf.structured has a weirdness when you try to apply + it to a dataclass whose first base class is a Generic which is not + Dict. The issue is with a function called get_dict_key_value_types + in omegaconf/_utils.py. + For example this fails: + + @dataclass(eq=False) + class D(torch.utils.data.Dataset[int]): + a: int = 3 + + OmegaConf.structured(D) + + We avoid the problem by adding this class as an extra base class. + """ + + pass + + +def is_known_frame_scalar(frame_type: str) -> bool: + """ + Given a single frame type corresponding to a single frame, return whether + the frame is a known frame. + """ + return frame_type.endswith(DATASET_TYPE_KNOWN) + + +def is_known_frame( + frame_type: List[str], device: Optional[str] = None +) -> torch.BoolTensor: + """ + Given a list `frame_type` of frame types in a batch, return a tensor + of boolean flags expressing whether the corresponding frame is a known frame. + """ + # pyre-fixme[7]: Expected `BoolTensor` but got `Tensor`. + return torch.tensor( + [is_known_frame_scalar(ft) for ft in frame_type], + dtype=torch.bool, + device=device, + ) + + +def is_train_frame( + frame_type: List[str], device: Optional[str] = None +) -> torch.BoolTensor: + """ + Given a list `frame_type` of frame types in a batch, return a tensor + of boolean flags expressing whether the corresponding frame is a training frame. + """ + # pyre-fixme[7]: Expected `BoolTensor` but got `Tensor`. + return torch.tensor( + [ft.startswith(DATASET_TYPE_TRAIN) for ft in frame_type], + dtype=torch.bool, + device=device, + ) + + +def get_bbox_from_mask( + mask: np.ndarray, thr: float, decrease_quant: float = 0.05 +) -> Tuple[int, int, int, int]: + # bbox in xywh + masks_for_box = np.zeros_like(mask) + while masks_for_box.sum() <= 1.0: + masks_for_box = (mask > thr).astype(np.float32) + thr -= decrease_quant + if thr <= 0.0: + warnings.warn( + f"Empty masks_for_bbox (thr={thr}) => using full image.", stacklevel=1 + ) + + x0, x1 = get_1d_bounds(masks_for_box.sum(axis=-2)) + y0, y1 = get_1d_bounds(masks_for_box.sum(axis=-1)) + + return x0, y0, x1 - x0, y1 - y0 + + +def crop_around_box( + tensor: torch.Tensor, bbox: torch.Tensor, impath: str = "" +) -> torch.Tensor: + # bbox is xyxy, where the upper bound is corrected with +1 + bbox = clamp_box_to_image_bounds_and_round( + bbox, + image_size_hw=tuple(tensor.shape[-2:]), + ) + tensor = tensor[..., bbox[1] : bbox[3], bbox[0] : bbox[2]] + assert all(c > 0 for c in tensor.shape), f"squashed image {impath}" + return tensor + + +def clamp_box_to_image_bounds_and_round( + bbox_xyxy: torch.Tensor, + image_size_hw: Tuple[int, int], +) -> torch.LongTensor: + bbox_xyxy = bbox_xyxy.clone() + bbox_xyxy[[0, 2]] = torch.clamp(bbox_xyxy[[0, 2]], 0, image_size_hw[-1]) + bbox_xyxy[[1, 3]] = torch.clamp(bbox_xyxy[[1, 3]], 0, image_size_hw[-2]) + if not isinstance(bbox_xyxy, torch.LongTensor): + bbox_xyxy = bbox_xyxy.round().long() + return bbox_xyxy # pyre-ignore [7] + + +T = TypeVar("T", bound=torch.Tensor) + + +def bbox_xyxy_to_xywh(xyxy: T) -> T: + wh = xyxy[2:] - xyxy[:2] + xywh = torch.cat([xyxy[:2], wh]) + return xywh # pyre-ignore + + +def get_clamp_bbox( + bbox: torch.Tensor, + box_crop_context: float = 0.0, + image_path: str = "", +) -> torch.Tensor: + # box_crop_context: rate of expansion for bbox + # returns possibly expanded bbox xyxy as float + + bbox = bbox.clone() # do not edit bbox in place + + # increase box size + if box_crop_context > 0.0: + c = box_crop_context + bbox = bbox.float() + bbox[0] -= bbox[2] * c / 2 + bbox[1] -= bbox[3] * c / 2 + bbox[2] += bbox[2] * c + bbox[3] += bbox[3] * c + + if (bbox[2:] <= 1.0).any(): + raise ValueError( + f"squashed image {image_path}!! The bounding box contains no pixels." + ) + + bbox[2:] = torch.clamp(bbox[2:], 2) # set min height, width to 2 along both axes + bbox_xyxy = bbox_xywh_to_xyxy(bbox, clamp_size=2) + + return bbox_xyxy + + +def rescale_bbox( + bbox: torch.Tensor, + orig_res: Union[Tuple[int, int], torch.LongTensor], + new_res: Union[Tuple[int, int], torch.LongTensor], +) -> torch.Tensor: + assert bbox is not None + assert np.prod(orig_res) > 1e-8 + # average ratio of dimensions + # pyre-ignore + rel_size = (new_res[0] / orig_res[0] + new_res[1] / orig_res[1]) / 2.0 + return bbox * rel_size + + +def bbox_xywh_to_xyxy( + xywh: torch.Tensor, clamp_size: Optional[int] = None +) -> torch.Tensor: + xyxy = xywh.clone() + if clamp_size is not None: + xyxy[2:] = torch.clamp(xyxy[2:], clamp_size) + xyxy[2:] += xyxy[:2] + return xyxy + + +def get_1d_bounds(arr: np.ndarray) -> Tuple[int, int]: + nz = np.flatnonzero(arr) + return nz[0], nz[-1] + 1 + + +def resize_image( + image: Union[np.ndarray, torch.Tensor], + image_height: Optional[int], + image_width: Optional[int], + mode: str = "bilinear", +) -> Tuple[torch.Tensor, float, torch.Tensor]: + + if isinstance(image, np.ndarray): + image = torch.from_numpy(image) + + if image_height is None or image_width is None: + # skip the resizing + return image, 1.0, torch.ones_like(image[:1]) + # takes numpy array or tensor, returns pytorch tensor + minscale = min( + image_height / image.shape[-2], + image_width / image.shape[-1], + ) + imre = torch.nn.functional.interpolate( + image[None], + scale_factor=minscale, + mode=mode, + align_corners=False if mode == "bilinear" else None, + recompute_scale_factor=True, + )[0] + imre_ = torch.zeros(image.shape[0], image_height, image_width) + imre_[:, 0 : imre.shape[1], 0 : imre.shape[2]] = imre + mask = torch.zeros(1, image_height, image_width) + mask[:, 0 : imre.shape[1], 0 : imre.shape[2]] = 1.0 + return imre_, minscale, mask + + +def transpose_normalize_image(image: np.ndarray) -> np.ndarray: + im = np.atleast_3d(image).transpose((2, 0, 1)) + return im.astype(np.float32) / 255.0 + + +def load_image(path: str) -> np.ndarray: + with Image.open(path) as pil_im: + im = np.array(pil_im.convert("RGB")) + + return transpose_normalize_image(im) + + +def load_mask(path: str) -> np.ndarray: + with Image.open(path) as pil_im: + mask = np.array(pil_im) + + return transpose_normalize_image(mask) + + +def load_depth(path: str, scale_adjustment: float) -> np.ndarray: + if path.lower().endswith(".exr"): + # NOTE: environment variable OPENCV_IO_ENABLE_OPENEXR must be set to 1 + # You will have to accept these vulnerabilities by using OpenEXR: + # https://github.com/opencv/opencv/issues/21326 + import cv2 + + d = cv2.imread(path, cv2.IMREAD_ANYCOLOR | cv2.IMREAD_ANYDEPTH)[..., 0] + d[d > 1e9] = 0.0 + elif path.lower().endswith(".png"): + d = load_16big_png_depth(path) + else: + raise ValueError('unsupported depth file name "%s"' % path) + + d = d * scale_adjustment + + d[~np.isfinite(d)] = 0.0 + return d[None] # fake feature channel + + +def load_16big_png_depth(depth_png: str) -> np.ndarray: + with Image.open(depth_png) as depth_pil: + # the image is stored with 16-bit depth but PIL reads it as I (32 bit). + # we cast it to uint16, then reinterpret as float16, then cast to float32 + depth = ( + np.frombuffer(np.array(depth_pil, dtype=np.uint16), dtype=np.float16) + .astype(np.float32) + .reshape((depth_pil.size[1], depth_pil.size[0])) + ) + return depth + + +def load_1bit_png_mask(file: str) -> np.ndarray: + with Image.open(file) as pil_im: + mask = (np.array(pil_im.convert("L")) > 0.0).astype(np.float32) + return mask + + +def load_depth_mask(path: str) -> np.ndarray: + if not path.lower().endswith(".png"): + raise ValueError('unsupported depth mask file name "%s"' % path) + m = load_1bit_png_mask(path) + return m[None] # fake feature channel + + +def safe_as_tensor(data, dtype): + return torch.tensor(data, dtype=dtype) if data is not None else None + + +def _convert_ndc_to_pixels( + focal_length: torch.Tensor, + principal_point: torch.Tensor, + image_size_wh: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + half_image_size = image_size_wh / 2 + rescale = half_image_size.min() + principal_point_px = half_image_size - principal_point * rescale + focal_length_px = focal_length * rescale + return focal_length_px, principal_point_px + + +def _convert_pixels_to_ndc( + focal_length_px: torch.Tensor, + principal_point_px: torch.Tensor, + image_size_wh: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + half_image_size = image_size_wh / 2 + rescale = half_image_size.min() + principal_point = (half_image_size - principal_point_px) / rescale + focal_length = focal_length_px / rescale + return focal_length, principal_point + + +def adjust_camera_to_bbox_crop_( + camera: PerspectiveCameras, + image_size_wh: torch.Tensor, + clamp_bbox_xywh: torch.Tensor, +) -> None: + if len(camera) != 1: + raise ValueError("Adjusting currently works with singleton cameras camera only") + + focal_length_px, principal_point_px = _convert_ndc_to_pixels( + camera.focal_length[0], + camera.principal_point[0], + image_size_wh, + ) + principal_point_px_cropped = principal_point_px - clamp_bbox_xywh[:2] + + focal_length, principal_point_cropped = _convert_pixels_to_ndc( + focal_length_px, + principal_point_px_cropped, + clamp_bbox_xywh[2:], + ) + + camera.focal_length = focal_length[None] + camera.principal_point = principal_point_cropped[None] + + +def adjust_camera_to_image_scale_( + camera: PerspectiveCameras, + original_size_wh: torch.Tensor, + new_size_wh: torch.LongTensor, +) -> PerspectiveCameras: + focal_length_px, principal_point_px = _convert_ndc_to_pixels( + camera.focal_length[0], + camera.principal_point[0], + original_size_wh, + ) + + # now scale and convert from pixels to NDC + image_size_wh_output = new_size_wh.float() + scale = (image_size_wh_output / original_size_wh).min(dim=-1, keepdim=True).values + focal_length_px_scaled = focal_length_px * scale + principal_point_px_scaled = principal_point_px * scale + + focal_length_scaled, principal_point_scaled = _convert_pixels_to_ndc( + focal_length_px_scaled, + principal_point_px_scaled, + image_size_wh_output, + ) + camera.focal_length = focal_length_scaled[None] + camera.principal_point = principal_point_scaled[None] # pyre-ignore + + +# NOTE this cache is per-worker; they are implemented as processes. +# each batch is loaded and collated by a single worker; +# since sequences tend to co-occur within batches, this is useful. +@functools.lru_cache(maxsize=256) +def load_pointcloud(pcl_path: Union[str, Path], max_points: int = 0) -> Pointclouds: + pcl = IO().load_pointcloud(pcl_path) + if max_points > 0: + pcl = pcl.subsample(max_points) + + return pcl diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/visualize.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/visualize.py new file mode 100644 index 0000000000000000000000000000000000000000..557f7b43d734ad4570dd3b279571379fa2a592fb --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/dataset/visualize.py @@ -0,0 +1,100 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from typing import cast, Optional, Tuple + +import torch +from pytorch3d.implicitron.tools.point_cloud_utils import get_rgbd_point_cloud +from pytorch3d.structures import Pointclouds + +from .frame_data import FrameData +from .json_index_dataset import JsonIndexDataset + + +def get_implicitron_sequence_pointcloud( + dataset: JsonIndexDataset, + sequence_name: Optional[str] = None, + mask_points: bool = True, + max_frames: int = -1, + num_workers: int = 0, + load_dataset_point_cloud: bool = False, +) -> Tuple[Pointclouds, FrameData]: + """ + Make a point cloud by sampling random points from each frame the dataset. + """ + + if len(dataset) == 0: + raise ValueError("The dataset is empty.") + + if not dataset.load_depths: + raise ValueError("The dataset has to load depths (dataset.load_depths=True).") + + if mask_points and not dataset.load_masks: + raise ValueError( + "For mask_points=True, the dataset has to load masks" + + " (dataset.load_masks=True)." + ) + + # setup the indices of frames loaded from the dataset db + sequence_entries = list(range(len(dataset))) + if sequence_name is not None: + sequence_entries = [ + ei + for ei in sequence_entries + # pyre-ignore[16] + if dataset.frame_annots[ei]["frame_annotation"].sequence_name + == sequence_name + ] + if len(sequence_entries) == 0: + raise ValueError( + f'There are no dataset entries for sequence name "{sequence_name}".' + ) + + # subsample loaded frames if needed + if (max_frames > 0) and (len(sequence_entries) > max_frames): + sequence_entries = [ + sequence_entries[i] + for i in torch.randperm(len(sequence_entries))[:max_frames].sort().values + ] + + # take only the part of the dataset corresponding to the sequence entries + sequence_dataset = torch.utils.data.Subset(dataset, sequence_entries) + + # load the required part of the dataset + loader = torch.utils.data.DataLoader( + sequence_dataset, + batch_size=len(sequence_dataset), + shuffle=False, + num_workers=num_workers, + collate_fn=dataset.frame_data_type.collate, + ) + + frame_data = next(iter(loader)) # there's only one batch + + # scene point cloud + if load_dataset_point_cloud: + if not dataset.load_point_clouds: + raise ValueError( + "For load_dataset_point_cloud=True, the dataset has to" + + " load point clouds (dataset.load_point_clouds=True)." + ) + point_cloud = frame_data.sequence_point_cloud + + else: + point_cloud = get_rgbd_point_cloud( + frame_data.camera, + frame_data.image_rgb, + frame_data.depth_map, + ( + (cast(torch.Tensor, frame_data.fg_probability) > 0.5).float() + if mask_points and frame_data.fg_probability is not None + else None + ), + ) + + return point_cloud, frame_data diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/eval_demo.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/eval_demo.py new file mode 100644 index 0000000000000000000000000000000000000000..9b0e14d7bdda2ac5f1dceb9cf36c4d50a1d66770 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/eval_demo.py @@ -0,0 +1,183 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +import dataclasses +import os +from enum import Enum +from typing import Any, cast, Dict, List, Optional, Tuple + +import lpips +import torch +from pytorch3d.implicitron.dataset.data_source import ImplicitronDataSource +from pytorch3d.implicitron.dataset.json_index_dataset import JsonIndexDataset +from pytorch3d.implicitron.dataset.json_index_dataset_map_provider import ( + CO3D_CATEGORIES, +) +from pytorch3d.implicitron.evaluation.evaluate_new_view_synthesis import ( + aggregate_nvs_results, + eval_batch, + pretty_print_nvs_metrics, + summarize_nvs_eval_results, +) +from pytorch3d.implicitron.models.model_dbir import ModelDBIR +from pytorch3d.implicitron.tools.utils import dataclass_to_cuda_ +from tqdm import tqdm + + +class Task(Enum): + SINGLE_SEQUENCE = "singlesequence" + MULTI_SEQUENCE = "multisequence" + + +def main() -> None: + """ + Evaluates new view synthesis metrics of a simple depth-based image rendering + (DBIR) model for multisequence/singlesequence tasks for several categories. + + The evaluation is conducted on the same data as in [1] and, hence, the results + are directly comparable to the numbers reported in [1]. + + References: + [1] J. Reizenstein, R. Shapovalov, P. Henzler, L. Sbordone, + P. Labatut, D. Novotny: + Common Objects in 3D: Large-Scale Learning + and Evaluation of Real-life 3D Category Reconstruction + """ + + task_results = {} + for task in (Task.SINGLE_SEQUENCE, Task.MULTI_SEQUENCE): + task_results[task] = [] + for category in CO3D_CATEGORIES[: (20 if task == Task.SINGLE_SEQUENCE else 10)]: + for single_sequence_id in ( + (0, 1) if task == Task.SINGLE_SEQUENCE else (None,) + ): + category_result = evaluate_dbir_for_category( + category, task=task, single_sequence_id=single_sequence_id + ) + print("") + print( + f"Results for task={task}; category={category};" + + ( + f" sequence={single_sequence_id}:" + if single_sequence_id is not None + else ":" + ) + ) + pretty_print_nvs_metrics(category_result) + print("") + + task_results[task].append(category_result) + _print_aggregate_results(task, task_results) + + for task in task_results: + _print_aggregate_results(task, task_results) + + +def evaluate_dbir_for_category( + category: str, + task: Task, + bg_color: Tuple[float, float, float] = (0.0, 0.0, 0.0), + single_sequence_id: Optional[int] = None, + num_workers: int = 16, +): + """ + Evaluates new view synthesis metrics of a simple depth-based image rendering + (DBIR) model for a given task, category, and sequence (in case task=='singlesequence'). + + Args: + category: Object category. + bg_color: Background color of the renders. + task: Evaluation task. Either singlesequence or multisequence. + single_sequence_id: The ID of the evaluiation sequence for the singlesequence task. + num_workers: The number of workers for the employed dataloaders. + path_manager: (optional) Used for interpreting paths. + + Returns: + category_result: A dictionary of quantitative metrics. + """ + + single_sequence_id = single_sequence_id if single_sequence_id is not None else -1 + + torch.manual_seed(42) + + dataset_map_provider_args = { + "category": category, + "dataset_root": os.environ["CO3D_DATASET_ROOT"], + "assert_single_seq": task == Task.SINGLE_SEQUENCE, + "task_str": task.value, + "test_on_train": False, + "test_restrict_sequence_id": single_sequence_id, + "dataset_JsonIndexDataset_args": {"load_point_clouds": True}, + } + data_source = ImplicitronDataSource( + dataset_map_provider_JsonIndexDatasetMapProvider_args=dataset_map_provider_args + ) + + datasets, dataloaders = data_source.get_datasets_and_dataloaders() + + test_dataset = datasets.test + test_dataloader = dataloaders.test + if test_dataset is None or test_dataloader is None: + raise ValueError("must have a test dataset.") + + image_size = cast(JsonIndexDataset, test_dataset).image_width + + if image_size is None: + raise ValueError("Image size should be set in the dataset") + + # init the simple DBIR model + model = ModelDBIR( + render_image_width=image_size, + render_image_height=image_size, + bg_color=bg_color, + max_points=int(1e5), + ) + model.cuda() + + # init the lpips model for eval + lpips_model = lpips.LPIPS(net="vgg") + lpips_model = lpips_model.cuda() + + per_batch_eval_results = [] + print("Evaluating DBIR model ...") + for frame_data in tqdm(test_dataloader): + frame_data = dataclass_to_cuda_(frame_data) + preds = model(**dataclasses.asdict(frame_data)) + per_batch_eval_results.append( + eval_batch( + frame_data, + preds["implicitron_render"], + bg_color=bg_color, + lpips_model=lpips_model, + ) + ) + + category_result_flat, category_result = summarize_nvs_eval_results( + per_batch_eval_results, + is_multisequence=task != Task.SINGLE_SEQUENCE, + ) + + return category_result["results"] + + +def _print_aggregate_results( + task: Task, task_results: Dict[Task, List[List[Dict[str, Any]]]] +) -> None: + """ + Prints the aggregate metrics for a given task. + """ + aggregate_task_result = aggregate_nvs_results(task_results[task]) + print("") + print(f"Aggregate results for task={task}:") + pretty_print_nvs_metrics(aggregate_task_result) + print("") + + +if __name__ == "__main__": + main() diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/evaluation/__init__.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/evaluation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ac1a72bde66f104691245d2de4e83c6863718d5 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/evaluation/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/evaluation/evaluate_new_view_synthesis.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/evaluation/evaluate_new_view_synthesis.py new file mode 100644 index 0000000000000000000000000000000000000000..05b893d7030719e344764a6b7f3f94c6bd2e0b09 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/evaluation/evaluate_new_view_synthesis.py @@ -0,0 +1,598 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +import copy +import warnings +from collections import OrderedDict +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Sequence, Tuple, TYPE_CHECKING, Union + +import numpy as np +import torch +import torch.nn.functional as F +from pytorch3d.implicitron.dataset.frame_data import FrameData +from pytorch3d.implicitron.dataset.utils import is_train_frame +from pytorch3d.implicitron.models.base_model import ImplicitronRender +from pytorch3d.implicitron.tools import vis_utils +from pytorch3d.implicitron.tools.image_utils import mask_background +from pytorch3d.implicitron.tools.metric_utils import calc_psnr, eval_depth, iou, rgb_l1 +from pytorch3d.implicitron.tools.point_cloud_utils import get_rgbd_point_cloud +from pytorch3d.implicitron.tools.vis_utils import make_depth_image +from pytorch3d.renderer.cameras import PerspectiveCameras +from pytorch3d.vis.plotly_vis import plot_scene +from tabulate import tabulate + +if TYPE_CHECKING: + from visdom import Visdom + + +EVAL_N_SRC_VIEWS = [1, 3, 5, 7, 9] + + +@dataclass +class _Visualizer: + image_render: torch.Tensor + image_rgb_masked: torch.Tensor + depth_render: torch.Tensor + depth_map: Optional[torch.Tensor] + depth_mask: Optional[torch.Tensor] + + visdom_env: str = "eval_debug" + + _viz: Optional["Visdom"] = field(init=False) + + def __post_init__(self): + self._viz = vis_utils.get_visdom_connection() + + def show_rgb( + self, loss_value: float, metric_name: str, loss_mask_now: torch.Tensor + ): + if self._viz is None: + return + self._viz.images( + torch.cat( + ( + self.image_render, + self.image_rgb_masked, + loss_mask_now.repeat(1, 3, 1, 1), + ), + dim=3, + ), + env=self.visdom_env, + win=metric_name, + opts={"title": f"{metric_name}_{loss_value:1.2f}"}, + ) + + def show_depth( + self, depth_loss: float, name_postfix: str, loss_mask_now: torch.Tensor + ): + if self._viz is None: + return + viz = self._viz + viz.images( + torch.cat( + (make_depth_image(self.depth_render, loss_mask_now),) + + ( + (make_depth_image(self.depth_map, loss_mask_now),) + if self.depth_map is not None + else () + ), + dim=3, + ), + env=self.visdom_env, + win="depth_abs" + name_postfix, + opts={"title": f"depth_abs_{name_postfix}_{depth_loss:1.2f}"}, + ) + viz.images( + loss_mask_now, + env=self.visdom_env, + win="depth_abs" + name_postfix + "_mask", + opts={"title": f"depth_abs_{name_postfix}_{depth_loss:1.2f}_mask"}, + ) + if self.depth_mask is not None: + viz.images( + self.depth_mask, + env=self.visdom_env, + win="depth_abs" + name_postfix + "_maskd", + opts={"title": f"depth_abs_{name_postfix}_{depth_loss:1.2f}_maskd"}, + ) + + # show the 3D plot + # pyre-fixme[9]: viewpoint_trivial has type `PerspectiveCameras`; used as + # `TensorProperties`. + viewpoint_trivial: PerspectiveCameras = PerspectiveCameras().to( + loss_mask_now.device + ) + _pcls = { + "pred_depth": get_rgbd_point_cloud( + viewpoint_trivial, + self.image_render, + self.depth_render, + # mask_crop, + torch.ones_like(self.depth_render), + # loss_mask_now, + ) + } + if self.depth_map is not None: + _pcls["gt_depth"] = get_rgbd_point_cloud( + viewpoint_trivial, + self.image_rgb_masked, + self.depth_map, + # mask_crop, + torch.ones_like(self.depth_map), + # loss_mask_now, + ) + + _pcls = {pn: p for pn, p in _pcls.items() if int(p.num_points_per_cloud()) > 0} + + plotlyplot = plot_scene( + {f"pcl{name_postfix}": _pcls}, # pyre-ignore + camera_scale=1.0, + pointcloud_max_points=10000, + pointcloud_marker_size=1, + ) + viz.plotlyplot( + plotlyplot, + env=self.visdom_env, + win=f"pcl{name_postfix}", + ) + + +def eval_batch( + frame_data: FrameData, + implicitron_render: ImplicitronRender, + bg_color: Union[torch.Tensor, Sequence, str, float] = "black", + mask_thr: float = 0.5, + lpips_model=None, + visualize: bool = False, + visualize_visdom_env: str = "eval_debug", + break_after_visualising: bool = True, +) -> Dict[str, Any]: + """ + Produce performance metrics for a single batch of new-view synthesis + predictions. + + Given a set of known views (for which frame_data.frame_type.endswith('known') + is True), a new-view synthesis method (NVS) is tasked to generate new views + of the scene from the viewpoint of the target views (for which + frame_data.frame_type.endswith('known') is False). The resulting + synthesized new views, stored in `implicitron_render`, are compared to the + target ground truth in `frame_data` in terms of geometry and appearance + resulting in a dictionary of metrics returned by the `eval_batch` function. + + Args: + frame_data: A FrameData object containing the input to the new view + synthesis method. + implicitron_render: The data describing the synthesized new views. + bg_color: The background color of the generated new views and the + ground truth. + lpips_model: A pre-trained model for evaluating the LPIPS metric. + visualize: If True, visualizes the results to Visdom. + + Returns: + results: A dictionary holding evaluation metrics. + + Throws: + ValueError if frame_data does not have frame_type, camera, or image_rgb + ValueError if the batch has a mix of training and test samples + ValueError if the batch frames are not [unseen, known, known, ...] + ValueError if one of the required fields in implicitron_render is missing + """ + frame_type = frame_data.frame_type + if frame_type is None: + raise ValueError("Frame type has not been set.") + + # we check that all those fields are not None but Pyre can't infer that properly + # TODO: assign to local variables and simplify the code. + if frame_data.image_rgb is None: + raise ValueError("Image is not in the evaluation batch.") + + if frame_data.camera is None: + raise ValueError("Camera is not in the evaluation batch.") + + # eval all results in the resolution of the frame_data image + image_resol = tuple(frame_data.image_rgb.shape[2:]) + + # Post-process the render: + # 1) check implicitron_render for Nones, + # 2) obtain copies to make sure we dont edit the original data, + # 3) take only the 1st (target) image + # 4) resize to match ground-truth resolution + cloned_render: Dict[str, torch.Tensor] = {} + for k in ["mask_render", "image_render", "depth_render"]: + field = getattr(implicitron_render, k) + if field is None: + raise ValueError(f"A required predicted field {k} is missing") + + imode = "bilinear" if k == "image_render" else "nearest" + cloned_render[k] = ( + F.interpolate(field[:1], size=image_resol, mode=imode).detach().clone() + ) + + frame_data = copy.deepcopy(frame_data) + + # mask the ground truth depth in case frame_data contains the depth mask + if frame_data.depth_map is not None and frame_data.depth_mask is not None: + frame_data.depth_map *= frame_data.depth_mask + + if not isinstance(frame_type, list): # not batch FrameData + frame_type = [frame_type] + + is_train = is_train_frame(frame_type) + if len(is_train) > 1 and (is_train[1] != is_train[1:]).any(): + raise ValueError( + "All (conditioning) frames in the eval batch have to be either train/test." + ) + + for k in [ + "depth_map", + "image_rgb", + "fg_probability", + "mask_crop", + ]: + if not hasattr(frame_data, k) or getattr(frame_data, k) is None: + continue + setattr(frame_data, k, getattr(frame_data, k)[:1]) + + if frame_data.depth_map is None or frame_data.depth_map.sum() <= 0: + warnings.warn("Empty or missing depth map in evaluation!") + + if frame_data.mask_crop is None: + warnings.warn("mask_crop is None, assuming the whole image is valid.") + + if frame_data.fg_probability is None: + warnings.warn("fg_probability is None, assuming the whole image is fg.") + + # threshold the masks to make ground truth binary masks + mask_fg = ( + frame_data.fg_probability >= mask_thr + if frame_data.fg_probability is not None + # pyre-ignore [16] + else torch.ones_like(frame_data.image_rgb[:, :1, ...]).bool() + ) + + mask_crop = ( + frame_data.mask_crop + if frame_data.mask_crop is not None + else torch.ones_like(mask_fg) + ) + + # unmasked g.t. image + image_rgb = frame_data.image_rgb + + # fg-masked g.t. image + image_rgb_masked = mask_background( + # pyre-fixme[6]: Expected `Tensor` for 1st param but got + # `Optional[torch.Tensor]`. + frame_data.image_rgb, + mask_fg, + bg_color=bg_color, + ) + + # clamp predicted images + image_render = cloned_render["image_render"].clamp(0.0, 1.0) + + if visualize: + visualizer = _Visualizer( + image_render=image_render, + image_rgb_masked=image_rgb_masked, + depth_render=cloned_render["depth_render"], + depth_map=frame_data.depth_map, + depth_mask=( + frame_data.depth_mask[:1] if frame_data.depth_mask is not None else None + ), + visdom_env=visualize_visdom_env, + ) + + results: Dict[str, Any] = {} + + results["iou"] = iou( + cloned_render["mask_render"], + mask_fg, + mask=mask_crop, + ) + + for loss_fg_mask, name_postfix in zip((mask_crop, mask_fg), ("_masked", "_fg")): + + loss_mask_now = mask_crop * loss_fg_mask + + for rgb_metric_name, rgb_metric_fun in zip( + ("psnr", "rgb_l1"), (calc_psnr, rgb_l1) + ): + metric_name = rgb_metric_name + name_postfix + results[metric_name] = rgb_metric_fun( + image_render, + image_rgb_masked, + mask=loss_mask_now, + ) + + if visualize: + visualizer.show_rgb( + results[metric_name].item(), metric_name, loss_mask_now + ) + + if name_postfix == "_fg" and frame_data.depth_map is not None: + # only record depth metrics for the foreground + _, abs_ = eval_depth( + cloned_render["depth_render"], + # pyre-fixme[6]: For 2nd param expected `Tensor` but got + # `Optional[Tensor]`. + frame_data.depth_map, + get_best_scale=True, + mask=loss_mask_now, + crop=5, + ) + results["depth_abs" + name_postfix] = abs_.mean() + + if visualize: + visualizer.show_depth(abs_.mean().item(), name_postfix, loss_mask_now) + if break_after_visualising: + breakpoint() # noqa: B601 + + # add the rgb metrics between the render and the unmasked image + for rgb_metric_name, rgb_metric_fun in zip( + ("psnr_full_image", "rgb_l1_full_image"), (calc_psnr, rgb_l1) + ): + results[rgb_metric_name] = rgb_metric_fun( + image_render, + # pyre-fixme[6]: For 2nd argument expected `Tensor` but got + # `Optional[Tensor]`. + image_rgb, + mask=mask_crop, + ) + + if lpips_model is not None: + for gt_image_type in ("_full_image", "_masked"): + im1, im2 = [ + 2.0 * im.clamp(0.0, 1.0) - 1.0 # pyre-ignore[16] + for im in ( + image_rgb_masked if gt_image_type == "_masked" else image_rgb, + cloned_render["image_render"], + ) + ] + results["lpips" + gt_image_type] = lpips_model.forward(im1, im2).item() + + # convert all metrics to floats + results = {k: float(v) for k, v in results.items()} + + results["meta"] = { + # store the size of the batch (corresponds to n_src_views+1) + "batch_size": len(frame_type), + # store the type of the target frame + # pyre-fixme[16]: `None` has no attribute `__getitem__`. + "frame_type": str(frame_data.frame_type[0]), + } + + return results + + +def average_per_batch_results( + results_per_batch: List[Dict[str, Any]], + idx: Optional[torch.Tensor] = None, +) -> dict: + """ + Average a list of per-batch metrics `results_per_batch`. + Optionally, if `idx` is given, only a subset of the per-batch + metrics, indexed by `idx`, is averaged. + """ + result_keys = list(results_per_batch[0].keys()) + result_keys.remove("meta") + if idx is not None: + results_per_batch = [results_per_batch[i] for i in idx] + if len(results_per_batch) == 0: + return {k: float("NaN") for k in result_keys} + return { + k: float(np.array([r[k] for r in results_per_batch]).mean()) + for k in result_keys + } + + +def _reduce_camera_iou_overlap(ious: torch.Tensor, topk: int = 2) -> torch.Tensor: + """ + Calculate the final camera difficulty by computing the average of the + ious of the two most similar cameras. + + Returns: + single-element Tensor + """ + return ious.topk(k=min(topk, len(ious) - 1)).values.mean() + + +def _get_camera_difficulty_bin_edges(camera_difficulty_bin_breaks: Tuple[float, float]): + """ + Get the edges of camera difficulty bins. + """ + _eps = 1e-5 + lower, upper = camera_difficulty_bin_breaks + diff_bin_edges = torch.tensor([0.0 - _eps, lower, upper, 1.0 + _eps]).float() + diff_bin_names = ["hard", "medium", "easy"] + return diff_bin_edges, diff_bin_names + + +def summarize_nvs_eval_results( + per_batch_eval_results: List[Dict[str, Any]], + is_multisequence: bool, +) -> Tuple[Dict[str, Any], Dict[str, Any]]: + """ + Compile the per-batch evaluation results `per_batch_eval_results` into + a set of aggregate metrics. The produced metrics depend on is_multisequence. + + Args: + per_batch_eval_results: Metrics of each per-batch evaluation. + is_multisequence: Whether to evaluate as a multisequence task + camera_difficulty_bin_breaks: edge hard-medium and medium-easy + + + Returns: + nvs_results_flat: A flattened dict of all aggregate metrics. + aux_out: A dictionary holding a set of auxiliary results. + """ + n_batches = len(per_batch_eval_results) + eval_sets: List[Optional[str]] = [] + eval_sets = [None] + if is_multisequence: + eval_sets = ["train", "test"] + batch_sizes = torch.tensor( + [r["meta"]["batch_size"] for r in per_batch_eval_results] + ).long() + + is_train = is_train_frame([r["meta"]["frame_type"] for r in per_batch_eval_results]) + + # init the result database dict + results = [] + + # add per set averages + for SET in eval_sets: + if SET is None: + ok_set = torch.ones(n_batches, dtype=torch.bool) + set_name = "test" + else: + ok_set = is_train == int(SET == "train") + set_name = SET + + # average over all results + bin_results = average_per_batch_results( + per_batch_eval_results, idx=torch.where(ok_set)[0] + ) + results.append( + { + "subset": set_name, + "subsubset": "diff=all", + "metrics": bin_results, + } + ) + + if is_multisequence: + # split based on n_src_views + n_src_views = batch_sizes - 1 + for n_src in EVAL_N_SRC_VIEWS: + ok_src = ok_set & (n_src_views == n_src) + n_src_results = average_per_batch_results( + per_batch_eval_results, + idx=torch.where(ok_src)[0], + ) + results.append( + { + "subset": set_name, + "subsubset": f"n_src={int(n_src)}", + "metrics": n_src_results, + } + ) + + aux_out = {"results": results} + return flatten_nvs_results(results), aux_out + + +def _get_flat_nvs_metric_key(result, metric_name) -> str: + metric_key_postfix = f"|subset={result['subset']}|{result['subsubset']}" + metric_key = f"{metric_name}{metric_key_postfix}" + return metric_key + + +def flatten_nvs_results(results) -> Dict[str, Any]: + """ + Takes input `results` list of dicts of the form:: + + [ + { + 'subset':'train/test/...', + 'subsubset': 'src=1/src=2/...', + 'metrics': nvs_eval_metrics} + }, + ... + ] + + And converts to a flat dict as follows:: + + { + 'subset=train/test/...|subsubset=src=1/src=2/...': nvs_eval_metrics, + ... + } + """ + results_flat = {} + for result in results: + for metric_name, metric_val in result["metrics"].items(): + metric_key = _get_flat_nvs_metric_key(result, metric_name) + assert metric_key not in results_flat + results_flat[metric_key] = metric_val + return results_flat + + +def pretty_print_nvs_metrics(results) -> None: + subsets, subsubsets = [ + _ordered_set([r[k] for r in results]) for k in ("subset", "subsubset") + ] + metrics = _ordered_set([metric for r in results for metric in r["metrics"]]) + + for subset in subsets: + tab = {} + for metric in metrics: + tab[metric] = [] + header = ["metric"] + for subsubset in subsubsets: + metric_vals = [ + r["metrics"][metric] + for r in results + if r["subsubset"] == subsubset and r["subset"] == subset + ] + if len(metric_vals) > 0: + tab[metric].extend(metric_vals) + header.extend(subsubsets) + + if any(len(v) > 0 for v in tab.values()): + print(f"===== NVS results; subset={subset} =====") + print( + tabulate( + [[metric, *v] for metric, v in tab.items()], + # pyre-fixme[61]: `header` is undefined, or not always defined. + headers=header, + ) + ) + + +def _ordered_set(list_): + return list(OrderedDict((i, 0) for i in list_).keys()) + + +def aggregate_nvs_results(task_results): + """ + Aggregate nvs results. + For singlescene, this averages over all categories and scenes, + for multiscene, the average is over all per-category results. + """ + task_results_cat = [r_ for r in task_results for r_ in r] + subsets, subsubsets = [ + _ordered_set([r[k] for r in task_results_cat]) for k in ("subset", "subsubset") + ] + metrics = _ordered_set( + [metric for r in task_results_cat for metric in r["metrics"]] + ) + average_results = [] + for subset in subsets: + for subsubset in subsubsets: + metrics_lists = [ + r["metrics"] + for r in task_results_cat + if r["subsubset"] == subsubset and r["subset"] == subset + ] + avg_metrics = {} + for metric in metrics: + avg_metrics[metric] = float( + np.nanmean( + np.array([metric_list[metric] for metric_list in metrics_lists]) + ) + ) + average_results.append( + { + "subset": subset, + "subsubset": subsubset, + "metrics": avg_metrics, + } + ) + return average_results diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/evaluation/evaluator.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/evaluation/evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..84306997cc414d8316cd62125570ff182ae3cbb0 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/evaluation/evaluator.py @@ -0,0 +1,160 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import copy +import json +import logging +import os +import warnings +from typing import Any, Dict, List, Optional, Tuple + +import torch + +import tqdm +from pytorch3d.implicitron.evaluation import evaluate_new_view_synthesis as evaluate +from pytorch3d.implicitron.models.base_model import EvaluationMode, ImplicitronModelBase +from pytorch3d.implicitron.tools.config import ( + registry, + ReplaceableBase, + run_auto_creation, +) +from torch.utils.data import DataLoader + +logger = logging.getLogger(__name__) + + +class EvaluatorBase(ReplaceableBase): + """ + Evaluate a trained model on given data. Returns a dict of loss/objective + names and their values. + """ + + is_multisequence: bool = False + + def run( + self, model: ImplicitronModelBase, dataloader: DataLoader, **kwargs + ) -> Dict[str, Any]: + """ + Evaluate the results of Implicitron training. + """ + raise NotImplementedError() + + +@registry.register +class ImplicitronEvaluator(EvaluatorBase): + """ + Evaluate the results of Implicitron training. + """ + + # UNUSED; preserved for compatibility purposes + camera_difficulty_bin_breaks: Tuple[float, ...] = 0.97, 0.98 + + def __post_init__(self): + run_auto_creation(self) + + # pyre-fixme[14]: `run` overrides method defined in `EvaluatorBase` inconsistently. + def run( + self, + model: ImplicitronModelBase, + dataloader: DataLoader, + device: torch.device, + dump_to_json: bool = False, + exp_dir: Optional[str] = None, + epoch: Optional[int] = None, + **kwargs, + ) -> Dict[str, Any]: + """ + Evaluate the results of Implicitron training. Optionally, dump results to + exp_dir/results_test.json. + + Args: + model: A (trained) model to evaluate. + dataloader: A test dataloader. + device: A torch device. + dump_to_json: If True, will dump the results to a json file. + exp_dir: Root expeirment directory. + epoch: Evaluation epoch (to be stored in the results dict). + + Returns: + A dictionary of results. + """ + try: + import lpips + + lpips_model = lpips.LPIPS(net="vgg") + lpips_model = lpips_model.to(device) + except ImportError: + warnings.warn( + "lpips library NOT FOUND. lpips losses will not be calculated" + ) + lpips_model = None + + model.eval() + + per_batch_eval_results = [] + logger.info("Evaluating model ...") + for frame_data in tqdm.tqdm(dataloader): + frame_data = frame_data.to(device) + + # mask out the unknown images so that the model does not see them + frame_data_for_eval = _get_eval_frame_data(frame_data) + + with torch.no_grad(): + preds = model( + **{ + **frame_data_for_eval, + "evaluation_mode": EvaluationMode.EVALUATION, + } + ) + implicitron_render = copy.deepcopy(preds["implicitron_render"]) + per_batch_eval_results.append( + evaluate.eval_batch( + frame_data, + implicitron_render, + bg_color="black", + lpips_model=lpips_model, + ) + ) + + _, category_result = evaluate.summarize_nvs_eval_results( + per_batch_eval_results, + self.is_multisequence, + ) + + results = category_result["results"] + evaluate.pretty_print_nvs_metrics(results) + if dump_to_json: + _dump_to_json(epoch, exp_dir, results) + + return category_result["results"] + + +def _dump_to_json( + epoch: Optional[int], exp_dir: Optional[str], results: List[Dict[str, Any]] +) -> None: + if epoch is not None: + for r in results: + r["eval_epoch"] = int(epoch) + logger.info("Evaluation results") + + if exp_dir is None: + raise ValueError("Cannot save results to json without a specified save path.") + with open(os.path.join(exp_dir, "results_test.json"), "w") as f: + json.dump(results, f) + + +def _get_eval_frame_data(frame_data: Any) -> Any: + """ + Masks the target image data to make sure we cannot use it at model evaluation + time. Assumes the first batch element is target, the rest are source. + """ + frame_data_for_eval = copy.deepcopy(frame_data) + for k in ("image_rgb", "depth_map", "fg_probability", "mask_crop"): + value = getattr(frame_data_for_eval, k) + value[0].zero_() + return frame_data_for_eval diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/__init__.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7dc294bfb87b1948f305f0291726535266035861 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/__init__.py @@ -0,0 +1,12 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +# Allows to register the models +# see: pytorch3d.implicitron.tools.config.registry:register +from pytorch3d.implicitron.models.generic_model import GenericModel +from pytorch3d.implicitron.models.overfit_model import OverfitModel diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/base_model.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/base_model.py new file mode 100644 index 0000000000000000000000000000000000000000..ff645fd709ed20f61e0b68064b47662acb0f839c --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/base_model.py @@ -0,0 +1,93 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +import torch + +from pytorch3d.implicitron.models.renderer.base import EvaluationMode +from pytorch3d.implicitron.tools.config import ReplaceableBase +from pytorch3d.renderer.cameras import CamerasBase + + +@dataclass +class ImplicitronRender: + """ + Holds the tensors that describe a result of rendering. + """ + + depth_render: Optional[torch.Tensor] = None + image_render: Optional[torch.Tensor] = None + mask_render: Optional[torch.Tensor] = None + camera_distance: Optional[torch.Tensor] = None + + def clone(self) -> "ImplicitronRender": + def safe_clone(t: Optional[torch.Tensor]) -> Optional[torch.Tensor]: + return t.detach().clone() if t is not None else None + + return ImplicitronRender( + depth_render=safe_clone(self.depth_render), + image_render=safe_clone(self.image_render), + mask_render=safe_clone(self.mask_render), + camera_distance=safe_clone(self.camera_distance), + ) + + +class ImplicitronModelBase(ReplaceableBase, torch.nn.Module): + """ + Replaceable abstract base for all image generation / rendering models. + `forward()` method produces a render with a depth map. Derives from Module + so we can rely on basic functionality provided to torch for model + optimization. + """ + + # The keys from `preds` (output of ImplicitronModelBase.forward) to be logged in + # the training loop. + log_vars: List[str] = field(default_factory=lambda: ["objective"]) + + def forward( + self, + *, # force keyword-only arguments + image_rgb: Optional[torch.Tensor], + camera: CamerasBase, + fg_probability: Optional[torch.Tensor], + mask_crop: Optional[torch.Tensor], + depth_map: Optional[torch.Tensor], + sequence_name: Optional[List[str]], + evaluation_mode: EvaluationMode = EvaluationMode.EVALUATION, + **kwargs, + ) -> Dict[str, Any]: + """ + Args: + image_rgb: A tensor of shape `(B, 3, H, W)` containing a batch of rgb images; + the first `min(B, n_train_target_views)` images are considered targets and + are used to supervise the renders; the rest corresponding to the source + viewpoints from which features will be extracted. + camera: An instance of CamerasBase containing a batch of `B` cameras corresponding + to the viewpoints of target images, from which the rays will be sampled, + and source images, which will be used for intersecting with target rays. + fg_probability: A tensor of shape `(B, 1, H, W)` containing a batch of + foreground masks. + mask_crop: A binary tensor of shape `(B, 1, H, W)` deonting valid + regions in the input images (i.e. regions that do not correspond + to, e.g., zero-padding). When the `RaySampler`'s sampling mode is set to + "mask_sample", rays will be sampled in the non zero regions. + depth_map: A tensor of shape `(B, 1, H, W)` containing a batch of depth maps. + sequence_name: A list of `B` strings corresponding to the sequence names + from which images `image_rgb` were extracted. They are used to match + target frames with relevant source frames. + evaluation_mode: one of EvaluationMode.TRAINING or + EvaluationMode.EVALUATION which determines the settings used for + rendering. + + Returns: + preds: A dictionary containing all outputs of the forward pass. All models should + output an instance of `ImplicitronRender` in `preds["implicitron_render"]`. + """ + raise NotImplementedError() diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/feature_extractor/__init__.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/feature_extractor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6c47703b66349df4c048e5040b931ff9b67a925b --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/feature_extractor/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from .feature_extractor import FeatureExtractorBase diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/feature_extractor/feature_extractor.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/feature_extractor/feature_extractor.py new file mode 100644 index 0000000000000000000000000000000000000000..fd71b1e20d718408be6d553d3d1d38c1347a990a --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/feature_extractor/feature_extractor.py @@ -0,0 +1,43 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from typing import Any, Dict, Optional + +import torch +from pytorch3d.implicitron.tools.config import ReplaceableBase + + +class FeatureExtractorBase(ReplaceableBase, torch.nn.Module): + """ + Base class for an extractor of a set of features from images. + """ + + def get_feat_dims(self) -> int: + """ + Returns: + total number of feature dimensions of the output. + (i.e. sum_i(dim_i)) + """ + raise NotImplementedError + + def forward( + self, + imgs: Optional[torch.Tensor], + masks: Optional[torch.Tensor] = None, + **kwargs, + ) -> Dict[Any, torch.Tensor]: + """ + Args: + imgs: A batch of input images of shape `(B, 3, H, W)`. + masks: A batch of input masks of shape `(B, 3, H, W)`. + + Returns: + out_feats: A dict `{f_i: t_i}` keyed by predicted feature names `f_i` + and their corresponding tensors `t_i` of shape `(B, dim_i, H_i, W_i)`. + """ + raise NotImplementedError diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/feature_extractor/resnet_feature_extractor.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/feature_extractor/resnet_feature_extractor.py new file mode 100644 index 0000000000000000000000000000000000000000..08c86489150b6577236e059d6b2ca90f8fc7beb9 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/feature_extractor/resnet_feature_extractor.py @@ -0,0 +1,214 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import logging +import math +from typing import Any, Dict, Optional, Tuple + +import torch +import torch.nn.functional as Fu +import torchvision +from pytorch3d.implicitron.tools.config import registry + +from . import FeatureExtractorBase + + +logger = logging.getLogger(__name__) + +MASK_FEATURE_NAME = "mask" +IMAGE_FEATURE_NAME = "image" + +_FEAT_DIMS = { + "resnet18": (64, 128, 256, 512), + "resnet34": (64, 128, 256, 512), + "resnet50": (256, 512, 1024, 2048), + "resnet101": (256, 512, 1024, 2048), + "resnet152": (256, 512, 1024, 2048), +} + +_RESNET_MEAN = [0.485, 0.456, 0.406] +_RESNET_STD = [0.229, 0.224, 0.225] + + +@registry.register +class ResNetFeatureExtractor(FeatureExtractorBase): + """ + Implements an image feature extractor. Depending on the settings allows + to extract: + - deep features: A CNN ResNet backbone from torchvision (with/without + pretrained weights) which extracts deep features. + - masks: Segmentation masks. + - images: Raw input RGB images. + + Settings: + name: name of the resnet backbone (from torchvision) + pretrained: If true, will load the pretrained weights + stages: List of stages from which to extract features. + Features from each stage are returned as key value + pairs in the forward function + normalize_image: If set will normalize the RGB values of + the image based on the Resnet mean/std + image_rescale: If not 1.0, this rescale factor will be + used to resize the image + first_max_pool: If set, a max pool layer is added after the first + convolutional layer + proj_dim: The number of output channels for the convolutional layers + l2_norm: If set, l2 normalization is applied to the extracted features + add_masks: If set, the masks will be saved in the output dictionary + add_images: If set, the images will be saved in the output dictionary + global_average_pool: If set, global average pooling step is performed + feature_rescale: If not 1.0, this rescale factor will be used to + rescale the output features + """ + + name: str = "resnet34" + pretrained: bool = True + stages: Tuple[int, ...] = (1, 2, 3, 4) + normalize_image: bool = True + image_rescale: float = 128 / 800.0 + first_max_pool: bool = True + proj_dim: int = 32 + l2_norm: bool = True + add_masks: bool = True + add_images: bool = True + global_average_pool: bool = False # this can simulate global/non-spacial features + feature_rescale: float = 1.0 + + def __post_init__(self): + if self.normalize_image: + # register buffers needed to normalize the image + for k, v in (("_resnet_mean", _RESNET_MEAN), ("_resnet_std", _RESNET_STD)): + self.register_buffer( + k, + torch.FloatTensor(v).view(1, 3, 1, 1), + persistent=False, + ) + + self._feat_dim = {} + + if len(self.stages) == 0: + # do not extract any resnet features + pass + else: + net = getattr(torchvision.models, self.name)(pretrained=self.pretrained) + if self.first_max_pool: + self.stem = torch.nn.Sequential( + net.conv1, net.bn1, net.relu, net.maxpool + ) + else: + self.stem = torch.nn.Sequential(net.conv1, net.bn1, net.relu) + self.max_stage = max(self.stages) + self.layers = torch.nn.ModuleList() + self.proj_layers = torch.nn.ModuleList() + for stage in range(self.max_stage): + stage_name = f"layer{stage+1}" + feature_name = self._get_resnet_stage_feature_name(stage) + if (stage + 1) in self.stages: + if ( + self.proj_dim > 0 + and _FEAT_DIMS[self.name][stage] > self.proj_dim + ): + proj = torch.nn.Conv2d( + _FEAT_DIMS[self.name][stage], + self.proj_dim, + 1, + 1, + bias=True, + ) + self._feat_dim[feature_name] = self.proj_dim + else: + proj = torch.nn.Identity() + self._feat_dim[feature_name] = _FEAT_DIMS[self.name][stage] + else: + proj = torch.nn.Identity() + self.proj_layers.append(proj) + self.layers.append(getattr(net, stage_name)) + + if self.add_masks: + self._feat_dim[MASK_FEATURE_NAME] = 1 + + if self.add_images: + self._feat_dim[IMAGE_FEATURE_NAME] = 3 + + logger.info(f"Feat extractor total dim = {self.get_feat_dims()}") + self.stages = set(self.stages) # convert to set for faster "in" + + def _get_resnet_stage_feature_name(self, stage) -> str: + return f"res_layer_{stage+1}" + + def _resnet_normalize_image(self, img: torch.Tensor) -> torch.Tensor: + return (img - self._resnet_mean) / self._resnet_std + + def get_feat_dims(self) -> int: + return sum(self._feat_dim.values()) + + def forward( + self, + imgs: Optional[torch.Tensor], + masks: Optional[torch.Tensor] = None, + **kwargs, + ) -> Dict[Any, torch.Tensor]: + """ + Args: + imgs: A batch of input images of shape `(B, 3, H, W)`. + masks: A batch of input masks of shape `(B, 3, H, W)`. + + Returns: + out_feats: A dict `{f_i: t_i}` keyed by predicted feature names `f_i` + and their corresponding tensors `t_i` of shape `(B, dim_i, H_i, W_i)`. + """ + + out_feats = {} + + imgs_input = imgs + if self.image_rescale != 1.0 and imgs_input is not None: + imgs_resized = Fu.interpolate( + imgs_input, + scale_factor=self.image_rescale, + mode="bilinear", + ) + else: + imgs_resized = imgs_input + + if len(self.stages) > 0: + assert imgs_resized is not None + + if self.normalize_image: + imgs_normed = self._resnet_normalize_image(imgs_resized) + else: + imgs_normed = imgs_resized + # is not a function. + feats = self.stem(imgs_normed) + for stage, (layer, proj) in enumerate(zip(self.layers, self.proj_layers)): + feats = layer(feats) + # just a sanity check below + assert feats.shape[1] == _FEAT_DIMS[self.name][stage] + if (stage + 1) in self.stages: + f = proj(feats) + if self.global_average_pool: + f = f.mean(dims=(2, 3)) + if self.l2_norm: + normfac = 1.0 / math.sqrt(len(self.stages)) + f = Fu.normalize(f, dim=1) * normfac + feature_name = self._get_resnet_stage_feature_name(stage) + out_feats[feature_name] = f + + if self.add_masks: + assert masks is not None + out_feats[MASK_FEATURE_NAME] = masks + + if self.add_images: + assert imgs_resized is not None + out_feats[IMAGE_FEATURE_NAME] = imgs_resized + + if self.feature_rescale != 1.0: + out_feats = {k: self.feature_rescale * f for k, f in out_feats.items()} + + # pyre-fixme[7]: Incompatible return type, expected `Dict[typing.Any, Tensor]` + # but got `Dict[typing.Any, float]` + return out_feats diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/generic_model.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/generic_model.py new file mode 100644 index 0000000000000000000000000000000000000000..06a8f2b7e53b7bedd19200261460beb08343658d --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/generic_model.py @@ -0,0 +1,772 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +# Note: The #noqa comments below are for unused imports of pluggable implementations +# which are part of implicitron. They ensure that the registry is prepopulated. + +import logging +from dataclasses import field +from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING, Union + +import torch +from omegaconf import DictConfig + +from pytorch3d.implicitron.models.base_model import ( + ImplicitronModelBase, + ImplicitronRender, +) +from pytorch3d.implicitron.models.feature_extractor import FeatureExtractorBase +from pytorch3d.implicitron.models.global_encoder.global_encoder import GlobalEncoderBase +from pytorch3d.implicitron.models.implicit_function.base import ImplicitFunctionBase +from pytorch3d.implicitron.models.metrics import ( + RegularizationMetricsBase, + ViewMetricsBase, +) + +from pytorch3d.implicitron.models.renderer.base import ( + BaseRenderer, + EvaluationMode, + ImplicitFunctionWrapper, + ImplicitronRayBundle, + RendererOutput, + RenderSamplingMode, +) +from pytorch3d.implicitron.models.renderer.ray_sampler import RaySamplerBase + +from pytorch3d.implicitron.models.utils import ( + apply_chunked, + chunk_generator, + log_loss_weights, + preprocess_input, + weighted_sum_losses, +) +from pytorch3d.implicitron.models.view_pooler.view_pooler import ViewPooler +from pytorch3d.implicitron.tools import vis_utils +from pytorch3d.implicitron.tools.config import ( + expand_args_fields, + registry, + run_auto_creation, +) + +from pytorch3d.implicitron.tools.rasterize_mc import rasterize_sparse_ray_bundle +from pytorch3d.renderer import utils as rend_utils +from pytorch3d.renderer.cameras import CamerasBase + + +if TYPE_CHECKING: + from visdom import Visdom +logger = logging.getLogger(__name__) + + +@registry.register +class GenericModel(ImplicitronModelBase): # pyre-ignore: 13 + """ + GenericModel is a wrapper for the neural implicit + rendering and reconstruction pipeline which consists + of the following sequence of 7 steps (steps 2–4 are normally + skipped in overfitting scenario, since conditioning on source views + does not add much information; otherwise they should be present altogether): + + + (1) Ray Sampling + ------------------ + Rays are sampled from an image grid based on the target view(s). + │_____________ + │ │ + │ ▼ + │ (2) Feature Extraction (optional) + │ ----------------------- + │ A feature extractor (e.g. a convolutional + │ neural net) is used to extract image features + │ from the source view(s). + │ │ + │ ▼ + │ (3) View Sampling (optional) + │ ------------------ + │ Image features are sampled at the 2D projections + │ of a set of 3D points along each of the sampled + │ target rays from (1). + │ │ + │ ▼ + │ (4) Feature Aggregation (optional) + │ ------------------ + │ Aggregate features and masks sampled from + │ image view(s) in (3). + │ │ + │____________▼ + │ + ▼ + (5) Implicit Function Evaluation + ------------------ + Evaluate the implicit function(s) at the sampled ray points + (optionally pass in the aggregated image features from (4)). + (also optionally pass in a global encoding from global_encoder). + │ + ▼ + (6) Rendering + ------------------ + Render the image into the target cameras by raymarching along + the sampled rays and aggregating the colors and densities + output by the implicit function in (5). + │ + ▼ + (7) Loss Computation + ------------------ + Compute losses based on the predicted target image(s). + + + The `forward` function of GenericModel executes + this sequence of steps. Currently, steps 1, 3, 4, 5, 6 + can be customized by intializing a subclass of the appropriate + baseclass and adding the newly created module to the registry. + Please see https://github.com/facebookresearch/pytorch3d/blob/main/projects/implicitron_trainer/README.md#custom-plugins + for more details on how to create and register a custom component. + + In the config .yaml files for experiments, the parameters below are + contained in the + `model_factory_ImplicitronModelFactory_args.model_GenericModel_args` + node. As GenericModel derives from ReplaceableBase, the input arguments are + parsed by the run_auto_creation function to initialize the + necessary member modules. Please see implicitron_trainer/README.md + for more details on this process. + + Args: + mask_images: Whether or not to mask the RGB image background given the + foreground mask (the `fg_probability` argument of `GenericModel.forward`) + mask_depths: Whether or not to mask the depth image background given the + foreground mask (the `fg_probability` argument of `GenericModel.forward`) + render_image_width: Width of the output image to render + render_image_height: Height of the output image to render + mask_threshold: If greater than 0.0, the foreground mask is + thresholded by this value before being applied to the RGB/Depth images + output_rasterized_mc: If True, visualize the Monte-Carlo pixel renders by + splatting onto an image grid. Default: False. + bg_color: RGB values for setting the background color of input image + if mask_images=True. Defaults to (0.0, 0.0, 0.0). Each renderer has its own + way to determine the background color of its output, unrelated to this. + num_passes: The specified implicit_function is initialized num_passes + times and run sequentially. + chunk_size_grid: The total number of points which can be rendered + per chunk. This is used to compute the number of rays used + per chunk when the chunked version of the renderer is used (in order + to fit rendering on all rays in memory) + render_features_dimensions: The number of output features to render. + Defaults to 3, corresponding to RGB images. + n_train_target_views: The number of cameras to render into at training + time; first `n_train_target_views` in the batch are considered targets, + the rest are sources. + sampling_mode_training: The sampling method to use during training. Must be + a value from the RenderSamplingMode Enum. + sampling_mode_evaluation: Same as above but for evaluation. + global_encoder_class_type: The name of the class to use for global_encoder, + which must be available in the registry. Or `None` to disable global encoder. + global_encoder: An instance of `GlobalEncoder`. This is used to generate an encoding + of the image (referred to as the global_code) that can be used to model aspects of + the scene such as multiple objects or morphing objects. It is up to the implicit + function definition how to use it, but the most typical way is to broadcast and + concatenate to the other inputs for the implicit function. + raysampler_class_type: The name of the raysampler class which is available + in the global registry. + raysampler: An instance of RaySampler which is used to emit + rays from the target view(s). + renderer_class_type: The name of the renderer class which is available in the global + registry. + renderer: A renderer class which inherits from BaseRenderer. This is used to + generate the images from the target view(s). + image_feature_extractor_class_type: If a str, constructs and enables + the `image_feature_extractor` object of this type. Or None if not needed. + image_feature_extractor: A module for extrating features from an input image. + view_pooler_enabled: If `True`, constructs and enables the `view_pooler` object. + This means features are sampled from the source image(s) + at the projected 2d locations of the sampled 3d ray points from the target + view(s), i.e. this activates step (3) above. + view_pooler: An instance of ViewPooler which is used for sampling of + image-based features at the 2D projections of a set + of 3D points and aggregating the sampled features. + implicit_function_class_type: The type of implicit function to use which + is available in the global registry. + implicit_function: An instance of ImplicitFunctionBase. The actual implicit functions + are initialised to be in self._implicit_functions. + view_metrics: An instance of ViewMetricsBase used to compute loss terms which + are independent of the model's parameters. + view_metrics_class_type: The type of view metrics to use, must be available in + the global registry. + regularization_metrics: An instance of RegularizationMetricsBase used to compute + regularization terms which can depend on the model's parameters. + regularization_metrics_class_type: The type of regularization metrics to use, + must be available in the global registry. + loss_weights: A dictionary with a {loss_name: weight} mapping; see documentation + for `ViewMetrics` class for available loss functions. + log_vars: A list of variable names which should be logged. + The names should correspond to a subset of the keys of the + dict `preds` output by the `forward` function. + """ # noqa: B950 + + mask_images: bool = True + mask_depths: bool = True + render_image_width: int = 400 + render_image_height: int = 400 + mask_threshold: float = 0.5 + output_rasterized_mc: bool = False + bg_color: Tuple[float, float, float] = (0.0, 0.0, 0.0) + num_passes: int = 1 + chunk_size_grid: int = 4096 + render_features_dimensions: int = 3 + tqdm_trigger_threshold: int = 16 + + n_train_target_views: int = 1 + sampling_mode_training: str = "mask_sample" + sampling_mode_evaluation: str = "full_grid" + + # ---- global encoder settings + global_encoder_class_type: Optional[str] = None + global_encoder: Optional[GlobalEncoderBase] + + # ---- raysampler + raysampler_class_type: str = "AdaptiveRaySampler" + raysampler: RaySamplerBase + + # ---- renderer configs + renderer_class_type: str = "MultiPassEmissionAbsorptionRenderer" + renderer: BaseRenderer + + # ---- image feature extractor settings + # (This is only created if view_pooler is enabled) + image_feature_extractor: Optional[FeatureExtractorBase] + image_feature_extractor_class_type: Optional[str] = None + # ---- view pooler settings + view_pooler_enabled: bool = False + view_pooler: Optional[ViewPooler] + + # ---- implicit function settings + implicit_function_class_type: str = "NeuralRadianceFieldImplicitFunction" + # This is just a model, never constructed. + # The actual implicit functions live in self._implicit_functions + implicit_function: ImplicitFunctionBase + + # ----- metrics + view_metrics: ViewMetricsBase + view_metrics_class_type: str = "ViewMetrics" + + regularization_metrics: RegularizationMetricsBase + regularization_metrics_class_type: str = "RegularizationMetrics" + + # ---- loss weights + loss_weights: Dict[str, float] = field( + default_factory=lambda: { + "loss_rgb_mse": 1.0, + "loss_prev_stage_rgb_mse": 1.0, + "loss_mask_bce": 0.0, + "loss_prev_stage_mask_bce": 0.0, + } + ) + + # ---- variables to be logged (logger automatically ignores if not computed) + log_vars: List[str] = field( + default_factory=lambda: [ + "loss_rgb_psnr_fg", + "loss_rgb_psnr", + "loss_rgb_mse", + "loss_rgb_huber", + "loss_depth_abs", + "loss_depth_abs_fg", + "loss_mask_neg_iou", + "loss_mask_bce", + "loss_mask_beta_prior", + "loss_eikonal", + "loss_density_tv", + "loss_depth_neg_penalty", + "loss_autodecoder_norm", + # metrics that are only logged in 2+stage renderes + "loss_prev_stage_rgb_mse", + "loss_prev_stage_rgb_psnr_fg", + "loss_prev_stage_rgb_psnr", + "loss_prev_stage_mask_bce", + # basic metrics + "objective", + "epoch", + "sec/it", + ] + ) + + @classmethod + def pre_expand(cls) -> None: + # use try/finally to bypass cinder's lazy imports + try: + from pytorch3d.implicitron.models.feature_extractor.resnet_feature_extractor import ( # noqa: F401, B950 + ResNetFeatureExtractor, + ) + from pytorch3d.implicitron.models.implicit_function.idr_feature_field import ( # noqa: F401, B950 + IdrFeatureField, + ) + from pytorch3d.implicitron.models.implicit_function.neural_radiance_field import ( # noqa: F401, B950 + NeRFormerImplicitFunction, + ) + from pytorch3d.implicitron.models.implicit_function.scene_representation_networks import ( # noqa: F401, B950 + SRNHyperNetImplicitFunction, + ) + from pytorch3d.implicitron.models.implicit_function.voxel_grid_implicit_function import ( # noqa: F401, B950 + VoxelGridImplicitFunction, + ) + from pytorch3d.implicitron.models.renderer.lstm_renderer import ( # noqa: F401 + LSTMRenderer, + ) + from pytorch3d.implicitron.models.renderer.multipass_ea import ( # noqa + MultiPassEmissionAbsorptionRenderer, + ) + from pytorch3d.implicitron.models.renderer.sdf_renderer import ( # noqa: F401 + SignedDistanceFunctionRenderer, + ) + finally: + pass + + def __post_init__(self): + if self.view_pooler_enabled: + if self.image_feature_extractor_class_type is None: + raise ValueError( + "image_feature_extractor must be present for view pooling." + ) + run_auto_creation(self) + + self._implicit_functions = self._construct_implicit_functions() + + log_loss_weights(self.loss_weights, logger) + + def forward( + self, + *, # force keyword-only arguments + image_rgb: Optional[torch.Tensor], + camera: CamerasBase, + fg_probability: Optional[torch.Tensor] = None, + mask_crop: Optional[torch.Tensor] = None, + depth_map: Optional[torch.Tensor] = None, + sequence_name: Optional[List[str]] = None, + frame_timestamp: Optional[torch.Tensor] = None, + evaluation_mode: EvaluationMode = EvaluationMode.EVALUATION, + **kwargs, + ) -> Dict[str, Any]: + """ + Args: + image_rgb: A tensor of shape `(B, 3, H, W)` containing a batch of rgb images; + the first `min(B, n_train_target_views)` images are considered targets and + are used to supervise the renders; the rest corresponding to the source + viewpoints from which features will be extracted. + camera: An instance of CamerasBase containing a batch of `B` cameras corresponding + to the viewpoints of target images, from which the rays will be sampled, + and source images, which will be used for intersecting with target rays. + fg_probability: A tensor of shape `(B, 1, H, W)` containing a batch of + foreground masks. + mask_crop: A binary tensor of shape `(B, 1, H, W)` denoting valid + regions in the input images (i.e. regions that do not correspond + to, e.g., zero-padding). When the `RaySampler`'s sampling mode is set to + "mask_sample", rays will be sampled in the non zero regions. + depth_map: A tensor of shape `(B, 1, H, W)` containing a batch of depth maps. + sequence_name: A list of `B` strings corresponding to the sequence names + from which images `image_rgb` were extracted. They are used to match + target frames with relevant source frames. + frame_timestamp: Optionally a tensor of shape `(B,)` containing a batch + of frame timestamps. + evaluation_mode: one of EvaluationMode.TRAINING or + EvaluationMode.EVALUATION which determines the settings used for + rendering. + + Returns: + preds: A dictionary containing all outputs of the forward pass including the + rendered images, depths, masks, losses and other metrics. + """ + image_rgb, fg_probability, depth_map = preprocess_input( + image_rgb, + fg_probability, + depth_map, + self.mask_images, + self.mask_depths, + self.mask_threshold, + self.bg_color, + ) + + # Obtain the batch size from the camera as this is the only required input. + batch_size = camera.R.shape[0] + + # Determine the number of target views, i.e. cameras we render into. + n_targets = ( + 1 + if evaluation_mode == EvaluationMode.EVALUATION + else ( + batch_size + if self.n_train_target_views <= 0 + else min(self.n_train_target_views, batch_size) + ) + ) + + # A helper function for selecting n_target first elements from the input + # where the latter can be None. + def safe_slice_targets( + tensor: Optional[Union[torch.Tensor, List[str]]], + ) -> Optional[Union[torch.Tensor, List[str]]]: + return None if tensor is None else tensor[:n_targets] + + # Select the target cameras. + target_cameras = camera[list(range(n_targets))] + + # Determine the used ray sampling mode. + sampling_mode = RenderSamplingMode( + self.sampling_mode_training + if evaluation_mode == EvaluationMode.TRAINING + else self.sampling_mode_evaluation + ) + + # (1) Sample rendering rays with the ray sampler. + # pyre-ignore[29] + ray_bundle: ImplicitronRayBundle = self.raysampler( + target_cameras, + evaluation_mode, + mask=( + mask_crop[:n_targets] + if mask_crop is not None + and sampling_mode == RenderSamplingMode.MASK_SAMPLE + else None + ), + ) + + # custom_args hold additional arguments to the implicit function. + custom_args = {} + + if self.image_feature_extractor is not None: + # (2) Extract features for the image + img_feats = self.image_feature_extractor(image_rgb, fg_probability) + else: + img_feats = None + + if self.view_pooler_enabled: + if sequence_name is None: + raise ValueError("sequence_name must be provided for view pooling") + assert img_feats is not None + + # (3-4) Sample features and masks at the ray points. + # Aggregate features from multiple views. + def curried_viewpooler(pts): + return self.view_pooler( + pts=pts, + seq_id_pts=sequence_name[:n_targets], + camera=camera, + seq_id_camera=sequence_name, + feats=img_feats, + masks=mask_crop, + ) + + custom_args["fun_viewpool"] = curried_viewpooler + + global_code = None + if self.global_encoder is not None: + global_code = self.global_encoder( # pyre-fixme[29] + sequence_name=safe_slice_targets(sequence_name), + frame_timestamp=safe_slice_targets(frame_timestamp), + ) + custom_args["global_code"] = global_code + + for func in self._implicit_functions: + func.bind_args(**custom_args) + + inputs_to_be_chunked = {} + if fg_probability is not None and self.renderer.requires_object_mask(): + sampled_fb_prob = rend_utils.ndc_grid_sample( + fg_probability[:n_targets], ray_bundle.xys, mode="nearest" + ) + inputs_to_be_chunked["object_mask"] = sampled_fb_prob > 0.5 + + # (5)-(6) Implicit function evaluation and Rendering + rendered = self._render( + ray_bundle=ray_bundle, + sampling_mode=sampling_mode, + evaluation_mode=evaluation_mode, + implicit_functions=self._implicit_functions, + inputs_to_be_chunked=inputs_to_be_chunked, + ) + + # Unbind the custom arguments to prevent pytorch from storing + # large buffers of intermediate results due to points in the + # bound arguments. + for func in self._implicit_functions: + func.unbind_args() + + # A dict to store losses as well as rendering results. + preds: Dict[str, Any] = {} + + preds.update( + self.view_metrics( + results=preds, + raymarched=rendered, + ray_bundle=ray_bundle, + image_rgb=safe_slice_targets(image_rgb), + depth_map=safe_slice_targets(depth_map), + fg_probability=safe_slice_targets(fg_probability), + mask_crop=safe_slice_targets(mask_crop), + ) + ) + + preds.update( + self.regularization_metrics( + results=preds, + model=self, + ) + ) + + if sampling_mode == RenderSamplingMode.MASK_SAMPLE: + if self.output_rasterized_mc: + # Visualize the monte-carlo pixel renders by splatting onto + # an image grid. + ( + preds["images_render"], + preds["depths_render"], + preds["masks_render"], + ) = rasterize_sparse_ray_bundle( + ray_bundle, + rendered.features, + (self.render_image_height, self.render_image_width), + rendered.depths, + masks=rendered.masks, + ) + elif sampling_mode == RenderSamplingMode.FULL_GRID: + preds["images_render"] = rendered.features.permute(0, 3, 1, 2) + preds["depths_render"] = rendered.depths.permute(0, 3, 1, 2) + preds["masks_render"] = rendered.masks.permute(0, 3, 1, 2) + + preds["implicitron_render"] = ImplicitronRender( + image_render=preds["images_render"], + depth_render=preds["depths_render"], + mask_render=preds["masks_render"], + ) + else: + raise AssertionError("Unreachable state") + + # (7) Compute losses + objective = self._get_objective(preds) + if objective is not None: + preds["objective"] = objective + + return preds + + def _get_objective(self, preds: Dict[str, torch.Tensor]) -> Optional[torch.Tensor]: + """ + A helper function to compute the overall loss as the dot product + of individual loss functions with the corresponding weights. + """ + return weighted_sum_losses(preds, self.loss_weights) + + def visualize( + self, + viz: Optional["Visdom"], + visdom_env_imgs: str, + preds: Dict[str, Any], + prefix: str, + ) -> None: + """ + Helper function to visualize the predictions generated + in the forward pass. + + Args: + viz: Visdom connection object + visdom_env_imgs: name of visdom environment for the images. + preds: predictions dict like returned by forward() + prefix: prepended to the names of images + """ + if viz is None or not viz.check_connection(): + logger.info("no visdom server! -> skipping batch vis") + return + + idx_image = 0 + title = f"{prefix}_im{idx_image}" + + vis_utils.visualize_basics(viz, preds, visdom_env_imgs, title=title) + + def _render( + self, + *, + ray_bundle: ImplicitronRayBundle, + inputs_to_be_chunked: Dict[str, torch.Tensor], + sampling_mode: RenderSamplingMode, + **kwargs, + ) -> RendererOutput: + """ + Args: + ray_bundle: A `ImplicitronRayBundle` object containing the parametrizations of the + sampled rendering rays. + inputs_to_be_chunked: A collection of tensor of shape `(B, _, H, W)`. E.g. + SignedDistanceFunctionRenderer requires "object_mask", shape + (B, 1, H, W), the silhouette of the object in the image. When + chunking, they are passed to the renderer as shape + `(B, _, chunksize)`. + sampling_mode: The sampling method to use. Must be a value from the + RenderSamplingMode Enum. + + Returns: + An instance of RendererOutput + """ + if sampling_mode == RenderSamplingMode.FULL_GRID and self.chunk_size_grid > 0: + return apply_chunked( + self.renderer, + chunk_generator( + self.chunk_size_grid, + ray_bundle, + inputs_to_be_chunked, + self.tqdm_trigger_threshold, + **kwargs, + ), + lambda batch: torch.cat(batch, dim=1).reshape( + *ray_bundle.lengths.shape[:-1], -1 + ), + ) + else: + # pyre-fixme[29]: `BaseRenderer` is not a function. + return self.renderer( + ray_bundle=ray_bundle, + **inputs_to_be_chunked, + **kwargs, + ) + + def _get_viewpooled_feature_dim(self) -> int: + if self.view_pooler is None: + return 0 + assert self.image_feature_extractor is not None + return self.view_pooler.get_aggregated_feature_dim( + self.image_feature_extractor.get_feat_dims() + ) + + @classmethod + def raysampler_tweak_args(cls, type, args: DictConfig) -> None: + """ + We don't expose certain fields of the raysampler because we want to set + them from our own members. + """ + del args["sampling_mode_training"] + del args["sampling_mode_evaluation"] + del args["image_width"] + del args["image_height"] + + def create_raysampler(self): + extra_args = { + "sampling_mode_training": self.sampling_mode_training, + "sampling_mode_evaluation": self.sampling_mode_evaluation, + "image_width": self.render_image_width, + "image_height": self.render_image_height, + } + raysampler_args = getattr( + self, "raysampler_" + self.raysampler_class_type + "_args" + ) + self.raysampler = registry.get(RaySamplerBase, self.raysampler_class_type)( + **raysampler_args, **extra_args + ) + + @classmethod + def renderer_tweak_args(cls, type, args: DictConfig) -> None: + """ + We don't expose certain fields of the renderer because we want to set + them based on other inputs. + """ + args.pop("render_features_dimensions", None) + args.pop("object_bounding_sphere", None) + + def create_renderer(self): + extra_args = {} + + if self.renderer_class_type == "SignedDistanceFunctionRenderer": + extra_args["render_features_dimensions"] = self.render_features_dimensions + if not hasattr(self.raysampler, "scene_extent"): + raise ValueError( + "SignedDistanceFunctionRenderer requires" + + " a raysampler that defines the 'scene_extent' field" + + " (this field is supported by, e.g., the adaptive raysampler - " + + " self.raysampler_class_type='AdaptiveRaySampler')." + ) + extra_args["object_bounding_sphere"] = self.raysampler.scene_extent + + renderer_args = getattr(self, "renderer_" + self.renderer_class_type + "_args") + self.renderer = registry.get(BaseRenderer, self.renderer_class_type)( + **renderer_args, **extra_args + ) + + def create_implicit_function(self) -> None: + """ + No-op called by run_auto_creation so that self.implicit_function + does not get created. __post_init__ creates the implicit function(s) + in wrappers explicitly in self._implicit_functions. + """ + pass + + @classmethod + def implicit_function_tweak_args(cls, type, args: DictConfig) -> None: + """ + We don't expose certain implicit_function fields because we want to set + them based on other inputs. + """ + args.pop("feature_vector_size", None) + args.pop("encoding_dim", None) + args.pop("latent_dim", None) + args.pop("latent_dim_hypernet", None) + args.pop("color_dim", None) + + def _construct_implicit_functions(self): + """ + After run_auto_creation has been called, the arguments + for each of the possible implicit function methods are + available. `GenericModel` arguments are first validated + based on the custom requirements for each specific + implicit function method. Then the required implicit + function(s) are initialized. + """ + extra_args = {} + global_encoder_dim = ( + 0 if self.global_encoder is None else self.global_encoder.get_encoding_dim() + ) + viewpooled_feature_dim = self._get_viewpooled_feature_dim() + + if self.implicit_function_class_type in ( + "NeuralRadianceFieldImplicitFunction", + "NeRFormerImplicitFunction", + ): + extra_args["latent_dim"] = viewpooled_feature_dim + global_encoder_dim + extra_args["color_dim"] = self.render_features_dimensions + + if self.implicit_function_class_type == "IdrFeatureField": + extra_args["feature_vector_size"] = self.render_features_dimensions + extra_args["encoding_dim"] = global_encoder_dim + + if self.implicit_function_class_type == "SRNImplicitFunction": + extra_args["latent_dim"] = viewpooled_feature_dim + global_encoder_dim + + # srn_hypernet preprocessing + if self.implicit_function_class_type == "SRNHyperNetImplicitFunction": + extra_args["latent_dim"] = viewpooled_feature_dim + extra_args["latent_dim_hypernet"] = global_encoder_dim + + # check that for srn, srn_hypernet, idr we have self.num_passes=1 + implicit_function_type = registry.get( + ImplicitFunctionBase, self.implicit_function_class_type + ) + expand_args_fields(implicit_function_type) + if self.num_passes != 1 and not implicit_function_type.allows_multiple_passes(): + raise ValueError( + self.implicit_function_class_type + + f"requires num_passes=1 not {self.num_passes}" + ) + + if implicit_function_type.requires_pooling_without_aggregation(): + if self.view_pooler_enabled and self.view_pooler.has_aggregation(): + raise ValueError( + "The chosen implicit function requires view pooling without aggregation." + ) + config_name = f"implicit_function_{self.implicit_function_class_type}_args" + config = getattr(self, config_name, None) + if config is None: + raise ValueError(f"{config_name} not present") + implicit_functions_list = [ + ImplicitFunctionWrapper(implicit_function_type(**config, **extra_args)) + for _ in range(self.num_passes) + ] + return torch.nn.ModuleList(implicit_functions_list) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/global_encoder/__init__.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/global_encoder/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ac1a72bde66f104691245d2de4e83c6863718d5 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/global_encoder/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/global_encoder/autodecoder.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/global_encoder/autodecoder.py new file mode 100644 index 0000000000000000000000000000000000000000..0d941bf6f64ab8a003662ad1dbe0cfe7c881a9fa --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/global_encoder/autodecoder.py @@ -0,0 +1,163 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import warnings +from collections import defaultdict +from typing import Dict, List, Optional, Union + +import torch +from pytorch3d.implicitron.tools.config import Configurable + + +class Autodecoder(Configurable, torch.nn.Module): + """ + Autodecoder which maps a list of integer or string keys to optimizable embeddings. + + Settings: + encoding_dim: Embedding dimension for the decoder. + n_instances: The maximum number of instances stored by the autodecoder. + init_scale: Scale factor for the initial autodecoder weights. + ignore_input: If `True`, optimizes a single code for any input. + """ + + encoding_dim: int = 0 + n_instances: int = 1 + init_scale: float = 1.0 + ignore_input: bool = False + + def __post_init__(self): + if self.n_instances <= 0: + raise ValueError(f"Invalid n_instances {self.n_instances}") + + self._autodecoder_codes = torch.nn.Embedding( + self.n_instances, + self.encoding_dim, + scale_grad_by_freq=True, + ) + with torch.no_grad(): + # weight has been initialised from Normal(0, 1) + self._autodecoder_codes.weight *= self.init_scale + + self._key_map = self._build_key_map() + # Make sure to register hooks for correct handling of saving/loading + # the module's _key_map. + self._register_load_state_dict_pre_hook(self._load_key_map_hook) + self._register_state_dict_hook(_save_key_map_hook) + + def _build_key_map( + self, key_map_dict: Optional[Dict[str, int]] = None + ) -> Dict[str, int]: + """ + Args: + key_map_dict: A dictionary used to initialize the key_map. + + Returns: + key_map: a dictionary of key: id pairs. + """ + # increments the counter when asked for a new value + key_map = defaultdict(iter(range(self.n_instances)).__next__) + if key_map_dict is not None: + # Assign all keys from the loaded key_map_dict to self._key_map. + # Since this is done in the original order, it should generate + # the same set of key:id pairs. We check this with an assert to be sure. + for x, x_id in key_map_dict.items(): + x_id_ = key_map[x] + assert x_id == x_id_ + return key_map + + def calculate_squared_encoding_norm(self) -> Optional[torch.Tensor]: + return (self._autodecoder_codes.weight**2).mean() + + def get_encoding_dim(self) -> int: + return self.encoding_dim + + def forward(self, x: Union[torch.LongTensor, List[str]]) -> Optional[torch.Tensor]: + """ + Args: + x: A batch of `N` identifiers. Either a long tensor of size + `(N,)` keys in [0, n_instances), or a list of `N` string keys that + are hashed to codes (without collisions). + + Returns: + codes: A tensor of shape `(N, self.encoding_dim)` containing the + key-specific autodecoder codes. + """ + if self.ignore_input: + x = ["singleton"] + + if isinstance(x[0], str): + try: + # pyre-fixme[9]: x has type `Union[List[str], LongTensor]`; used as + # `Tensor`. + x = torch.tensor( + [self._key_map[elem] for elem in x], + dtype=torch.long, + device=next(self.parameters()).device, + ) + except StopIteration: + raise ValueError("Not enough n_instances in the autodecoder") from None + + return self._autodecoder_codes(x) + + def _load_key_map_hook( + self, + state_dict, + prefix, + local_metadata, + strict, + missing_keys, + unexpected_keys, + error_msgs, + ): + """ + Args: + state_dict (dict): a dict containing parameters and + persistent buffers. + prefix (str): the prefix for parameters and buffers used in this + module + local_metadata (dict): a dict containing the metadata for this module. + strict (bool): whether to strictly enforce that the keys in + :attr:`state_dict` with :attr:`prefix` match the names of + parameters and buffers in this module + missing_keys (list of str): if ``strict=True``, add missing keys to + this list + unexpected_keys (list of str): if ``strict=True``, add unexpected + keys to this list + error_msgs (list of str): error messages should be added to this + list, and will be reported together in + :meth:`~torch.nn.Module.load_state_dict` + + Returns: + Constructed key_map if it exists in the state_dict + else raises a warning only. + """ + key_map_key = prefix + "_key_map" + if key_map_key in state_dict: + key_map_dict = state_dict.pop(key_map_key) + self._key_map = self._build_key_map(key_map_dict=key_map_dict) + else: + warnings.warn("No key map in Autodecoder state dict!") + + +def _save_key_map_hook( + self, + state_dict, + prefix, + local_metadata, +) -> None: + """ + Args: + state_dict (dict): a dict containing parameters and + persistent buffers. + prefix (str): the prefix for parameters and buffers used in this + module + local_metadata (dict): a dict containing the metadata for this module. + """ + key_map_key = prefix + "_key_map" + key_map_dict = dict(self._key_map.items()) + state_dict[key_map_key] = key_map_dict diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/global_encoder/global_encoder.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/global_encoder/global_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..0f62a0b729b6b2affe24fa02fb21ddc31ade902c --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/global_encoder/global_encoder.py @@ -0,0 +1,127 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from typing import List, Optional, Union + +import torch +from pytorch3d.implicitron.tools.config import ( + registry, + ReplaceableBase, + run_auto_creation, +) +from pytorch3d.renderer.implicit import HarmonicEmbedding + +from .autodecoder import Autodecoder + + +class GlobalEncoderBase(ReplaceableBase): + """ + A base class for implementing encoders of global frame-specific quantities. + + The latter includes e.g. the harmonic encoding of a frame timestamp + (`HarmonicTimeEncoder`), or an autodecoder encoding of the frame's sequence + (`SequenceAutodecoder`). + """ + + def get_encoding_dim(self): + """ + Returns the dimensionality of the returned encoding. + """ + raise NotImplementedError() + + def calculate_squared_encoding_norm(self) -> Optional[torch.Tensor]: + """ + Calculates the squared norm of the encoding to report as the + `autodecoder_norm` loss of the model, as a zero dimensional tensor. + """ + raise NotImplementedError() + + def forward( + self, + *, + frame_timestamp: Optional[torch.Tensor] = None, + sequence_name: Optional[Union[torch.LongTensor, List[str]]] = None, + **kwargs, + ) -> torch.Tensor: + """ + Given a set of inputs to encode, generates a tensor containing the encoding. + + Returns: + encoding: The tensor containing the global encoding. + """ + raise NotImplementedError() + + +# TODO: probabilistic embeddings? +@registry.register +class SequenceAutodecoder(GlobalEncoderBase, torch.nn.Module): # pyre-ignore: 13 + """ + A global encoder implementation which provides an autodecoder encoding + of the frame's sequence identifier. + """ + + autodecoder: Autodecoder + + def __post_init__(self): + run_auto_creation(self) + + def get_encoding_dim(self): + return self.autodecoder.get_encoding_dim() + + def forward( + self, + *, + frame_timestamp: Optional[torch.Tensor] = None, + sequence_name: Optional[Union[torch.LongTensor, List[str]]] = None, + **kwargs, + ) -> torch.Tensor: + if sequence_name is None: + raise ValueError("sequence_name must be provided.") + # run dtype checks and pass sequence_name to self.autodecoder + return self.autodecoder(sequence_name) + + def calculate_squared_encoding_norm(self) -> Optional[torch.Tensor]: + return self.autodecoder.calculate_squared_encoding_norm() + + +@registry.register +class HarmonicTimeEncoder(GlobalEncoderBase, torch.nn.Module): + """ + A global encoder implementation which provides harmonic embeddings + of each frame's timestamp. + """ + + n_harmonic_functions: int = 10 + append_input: bool = True + time_divisor: float = 1.0 + + def __post_init__(self): + self._harmonic_embedding = HarmonicEmbedding( + n_harmonic_functions=self.n_harmonic_functions, + append_input=self.append_input, + ) + + def get_encoding_dim(self): + return self._harmonic_embedding.get_output_dim(1) + + def forward( + self, + *, + frame_timestamp: Optional[torch.Tensor] = None, + sequence_name: Optional[Union[torch.LongTensor, List[str]]] = None, + **kwargs, + ) -> torch.Tensor: + if frame_timestamp is None: + raise ValueError("frame_timestamp must be provided.") + if frame_timestamp.shape[-1] != 1: + raise ValueError("Frame timestamp's last dimensions should be one.") + time = frame_timestamp / self.time_divisor + return self._harmonic_embedding(time) + + def calculate_squared_encoding_norm(self) -> Optional[torch.Tensor]: + return None diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/__init__.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ac1a72bde66f104691245d2de4e83c6863718d5 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/base.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/base.py new file mode 100644 index 0000000000000000000000000000000000000000..d8d182dd6ee1fb1d41b87af27529902053c7f3dd --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/base.py @@ -0,0 +1,51 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from abc import ABC, abstractmethod +from typing import Optional + +from pytorch3d.implicitron.models.renderer.base import ImplicitronRayBundle + +from pytorch3d.implicitron.tools.config import ReplaceableBase +from pytorch3d.renderer.cameras import CamerasBase + + +class ImplicitFunctionBase(ABC, ReplaceableBase): + @abstractmethod + def forward( + self, + *, + ray_bundle: ImplicitronRayBundle, + fun_viewpool=None, + camera: Optional[CamerasBase] = None, + global_code=None, + **kwargs, + ): + raise NotImplementedError() + + @staticmethod + def allows_multiple_passes() -> bool: + """ + Returns True if this implicit function allows + multiple passes. + """ + return False + + @staticmethod + def requires_pooling_without_aggregation() -> bool: + """ + Returns True if this implicit function needs + pooling without aggregation. + """ + return False + + def on_bind_args(self) -> None: + """ + Called when the custom args are fixed in the main model forward pass. + """ + pass diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/decoding_functions.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/decoding_functions.py new file mode 100644 index 0000000000000000000000000000000000000000..437cdbddd406f512ee23ab8d8ebfdb074bca006f --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/decoding_functions.py @@ -0,0 +1,491 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +""" +This file contains + - modules which get used by ImplicitFunction objects for decoding an embedding defined in + space, e.g. to color or opacity. + - DecoderFunctionBase and its subclasses, which wrap some of those modules, providing + some such modules as an extension point which an ImplicitFunction object could use. +""" + +import logging +from dataclasses import field + +from enum import Enum +from typing import Dict, Optional, Tuple + +import torch + +from omegaconf import DictConfig + +from pytorch3d.implicitron.tools.config import ( + Configurable, + registry, + ReplaceableBase, + run_auto_creation, +) + +logger = logging.getLogger(__name__) + + +class DecoderActivation(Enum): + RELU = "relu" + SOFTPLUS = "softplus" + SIGMOID = "sigmoid" + IDENTITY = "identity" + + +class DecoderFunctionBase(ReplaceableBase, torch.nn.Module): + """ + Decoding function is a torch.nn.Module which takes the embedding of a location in + space and transforms it into the required quantity (for example density and color). + """ + + def forward( + self, features: torch.Tensor, z: Optional[torch.Tensor] = None + ) -> torch.Tensor: + """ + Args: + features (torch.Tensor): tensor of shape (batch, ..., num_in_features) + z: optional tensor to append to parts of the decoding function + Returns: + decoded_features (torch.Tensor) : tensor of + shape (batch, ..., num_out_features) + """ + raise NotImplementedError() + + +@registry.register +class ElementwiseDecoder(DecoderFunctionBase): + """ + Decoding function which scales the input, adds shift and then applies + `relu`, `softplus`, `sigmoid` or nothing on its input: + `result = operation(input * scale + shift)` + + Members: + scale: a scalar with which input is multiplied before being shifted. + Defaults to 1. + shift: a scalar which is added to the scaled input before performing + the operation. Defaults to 0. + operation: which operation to perform on the transformed input. Options are: + `RELU`, `SOFTPLUS`, `SIGMOID` or `IDENTITY`. Defaults to `IDENTITY`. + """ + + scale: float = 1 + shift: float = 0 + operation: DecoderActivation = DecoderActivation.IDENTITY + + def __post_init__(self): + if self.operation not in [ + DecoderActivation.RELU, + DecoderActivation.SOFTPLUS, + DecoderActivation.SIGMOID, + DecoderActivation.IDENTITY, + ]: + raise ValueError( + "`operation` can only be `RELU`, `SOFTPLUS`, `SIGMOID` or `IDENTITY`." + ) + + def forward( + self, features: torch.Tensor, z: Optional[torch.Tensor] = None + ) -> torch.Tensor: + transfomed_input = features * self.scale + self.shift + if self.operation == DecoderActivation.SOFTPLUS: + return torch.nn.functional.softplus(transfomed_input) + if self.operation == DecoderActivation.RELU: + return torch.nn.functional.relu(transfomed_input) + if self.operation == DecoderActivation.SIGMOID: + return torch.nn.functional.sigmoid(transfomed_input) + return transfomed_input + + +class MLPWithInputSkips(Configurable, torch.nn.Module): + """ + Implements the multi-layer perceptron architecture of the Neural Radiance Field. + + As such, `MLPWithInputSkips` is a multi layer perceptron consisting + of a sequence of linear layers with ReLU activations. + + Additionally, for a set of predefined layers `input_skips`, the forward pass + appends a skip tensor `z` to the output of the preceding layer. + + Note that this follows the architecture described in the Supplementary + Material (Fig. 7) of [1], for which keep the defaults for: + - `last_layer_bias_init` to None + - `last_activation` to "relu" + - `use_xavier_init` to `true` + + If you want to use this as a part of the color prediction in TensoRF model set: + - `last_layer_bias_init` to 0 + - `last_activation` to "sigmoid" + - `use_xavier_init` to `False` + + References: + [1] Ben Mildenhall and Pratul P. Srinivasan and Matthew Tancik + and Jonathan T. Barron and Ravi Ramamoorthi and Ren Ng: + NeRF: Representing Scenes as Neural Radiance Fields for View + Synthesis, ECCV2020 + + Members: + n_layers: The number of linear layers of the MLP. + input_dim: The number of channels of the input tensor. + output_dim: The number of channels of the output. + skip_dim: The number of channels of the tensor `z` appended when + evaluating the skip layers. + hidden_dim: The number of hidden units of the MLP. + input_skips: The list of layer indices at which we append the skip + tensor `z`. + last_layer_bias_init: If set then all the biases in the last layer + are initialized to that value. + last_activation: Which activation to use in the last layer. Options are: + "relu", "softplus", "sigmoid" and "identity". Default is "relu". + use_xavier_init: If True uses xavier init for all linear layer weights. + Otherwise the default PyTorch initialization is used. Default True. + """ + + n_layers: int = 8 + input_dim: int = 39 + output_dim: int = 256 + skip_dim: int = 39 + hidden_dim: int = 256 + input_skips: Tuple[int, ...] = (5,) + skip_affine_trans: bool = False + last_layer_bias_init: Optional[float] = None + last_activation: DecoderActivation = DecoderActivation.RELU + use_xavier_init: bool = True + + def __post_init__(self): + try: + last_activation = { + DecoderActivation.RELU: torch.nn.ReLU(True), + DecoderActivation.SOFTPLUS: torch.nn.Softplus(), + DecoderActivation.SIGMOID: torch.nn.Sigmoid(), + DecoderActivation.IDENTITY: torch.nn.Identity(), + }[self.last_activation] + except KeyError as e: + raise ValueError( + "`last_activation` can only be `RELU`," + " `SOFTPLUS`, `SIGMOID` or `IDENTITY`." + ) from e + + layers = [] + skip_affine_layers = [] + for layeri in range(self.n_layers): + dimin = self.hidden_dim if layeri > 0 else self.input_dim + dimout = self.hidden_dim if layeri + 1 < self.n_layers else self.output_dim + + if layeri > 0 and layeri in self.input_skips: + if self.skip_affine_trans: + skip_affine_layers.append( + self._make_affine_layer(self.skip_dim, self.hidden_dim) + ) + else: + dimin = self.hidden_dim + self.skip_dim + + linear = torch.nn.Linear(dimin, dimout) + if self.use_xavier_init: + _xavier_init(linear) + if layeri == self.n_layers - 1 and self.last_layer_bias_init is not None: + torch.nn.init.constant_(linear.bias, self.last_layer_bias_init) + layers.append( + torch.nn.Sequential(linear, torch.nn.ReLU(True)) + if not layeri + 1 < self.n_layers + else torch.nn.Sequential(linear, last_activation) + ) + self.mlp = torch.nn.ModuleList(layers) + if self.skip_affine_trans: + self.skip_affines = torch.nn.ModuleList(skip_affine_layers) + self._input_skips = set(self.input_skips) + self._skip_affine_trans = self.skip_affine_trans + + def _make_affine_layer(self, input_dim, hidden_dim): + l1 = torch.nn.Linear(input_dim, hidden_dim * 2) + l2 = torch.nn.Linear(hidden_dim * 2, hidden_dim * 2) + if self.use_xavier_init: + _xavier_init(l1) + _xavier_init(l2) + return torch.nn.Sequential(l1, torch.nn.ReLU(True), l2) + + def _apply_affine_layer(self, layer, x, z): + mu_log_std = layer(z) + mu, log_std = mu_log_std.split(mu_log_std.shape[-1] // 2, dim=-1) + std = torch.nn.functional.softplus(log_std) + return (x - mu) * std + + def forward(self, x: torch.Tensor, z: Optional[torch.Tensor] = None): + """ + Args: + x: The input tensor of shape `(..., input_dim)`. + z: The input skip tensor of shape `(..., skip_dim)` which is appended + to layers whose indices are specified by `input_skips`. + Returns: + y: The output tensor of shape `(..., output_dim)`. + """ + y = x + if z is None: + # if the skip tensor is None, we use `x` instead. + z = x + skipi = 0 + for li, layer in enumerate(self.mlp): + if li in self._input_skips: + if self._skip_affine_trans: + y = self._apply_affine_layer(self.skip_affines[skipi], y, z) + else: + y = torch.cat((y, z), dim=-1) + skipi += 1 + y = layer(y) + return y + + +@registry.register +# pyre-fixme[13]: Attribute `network` is never initialized. +class MLPDecoder(DecoderFunctionBase): + """ + Decoding function which uses `MLPWithIputSkips` to convert the embedding to output. + The `input_dim` of the `network` is set from the value of `input_dim` member. + + Members: + input_dim: dimension of input. + param_groups: dictionary where keys are names of individual parameters + or module members and values are the parameter group where the + parameter/member will be sorted to. "self" key is used to denote the + parameter group at the module level. Possible keys, including the "self" key + do not have to be defined. By default all parameters are put into "default" + parameter group and have the learning rate defined in the optimizer, + it can be overridden at the: + - module level with “self” key, all the parameters and child + module's parameters will be put to that parameter group + - member level, which is the same as if the `param_groups` in that + member has key=“self” and value equal to that parameter group. + This is useful if members do not have `param_groups`, for + example torch.nn.Linear. + - parameter level, parameter with the same name as the key + will be put to that parameter group. + network_args: configuration for MLPWithInputSkips + """ + + input_dim: int = 3 + param_groups: Dict[str, str] = field(default_factory=lambda: {}) + network: MLPWithInputSkips + + def __post_init__(self): + run_auto_creation(self) + + def forward( + self, features: torch.Tensor, z: Optional[torch.Tensor] = None + ) -> torch.Tensor: + return self.network(features, z) + + @classmethod + def network_tweak_args(cls, type, args: DictConfig) -> None: + """ + Special method to stop get_default_args exposing member's `input_dim`. + """ + args.pop("input_dim", None) + + def create_network_impl(self, type, args: DictConfig) -> None: + """ + Set the input dimension of the `network` to the input dimension of the + decoding function. + """ + self.network = MLPWithInputSkips(input_dim=self.input_dim, **args) + + +class TransformerWithInputSkips(torch.nn.Module): + def __init__( + self, + n_layers: int = 8, + input_dim: int = 39, + output_dim: int = 256, + skip_dim: int = 39, + hidden_dim: int = 64, + input_skips: Tuple[int, ...] = (5,), + dim_down_factor: float = 1, + ): + """ + Args: + n_layers: The number of linear layers of the MLP. + input_dim: The number of channels of the input tensor. + output_dim: The number of channels of the output. + skip_dim: The number of channels of the tensor `z` appended when + evaluating the skip layers. + hidden_dim: The number of hidden units of the MLP. + input_skips: The list of layer indices at which we append the skip + tensor `z`. + """ + super().__init__() + + self.first = torch.nn.Linear(input_dim, hidden_dim) + _xavier_init(self.first) + + self.skip_linear = torch.nn.ModuleList() + + layers_pool, layers_ray = [], [] + dimout = 0 + for layeri in range(n_layers): + dimin = int(round(hidden_dim / (dim_down_factor**layeri))) + dimout = int(round(hidden_dim / (dim_down_factor ** (layeri + 1)))) + logger.info(f"Tr: {dimin} -> {dimout}") + for _i, l in enumerate((layers_pool, layers_ray)): + l.append( + TransformerEncoderLayer( + d_model=[dimin, dimout][_i], + nhead=4, + dim_feedforward=hidden_dim, + dropout=0.0, + d_model_out=dimout, + ) + ) + + if layeri in input_skips: + self.skip_linear.append(torch.nn.Linear(input_dim, dimin)) + + self.last = torch.nn.Linear(dimout, output_dim) + _xavier_init(self.last) + + # pyre-fixme[8]: Attribute has type `Tuple[ModuleList, ModuleList]`; used as + # `ModuleList`. + self.layers_pool, self.layers_ray = ( + torch.nn.ModuleList(layers_pool), + torch.nn.ModuleList(layers_ray), + ) + self._input_skips = set(input_skips) + + def forward( + self, + x: torch.Tensor, + z: Optional[torch.Tensor] = None, + ): + """ + Args: + x: The input tensor of shape + `(minibatch, n_pooled_feats, ..., n_ray_pts, input_dim)`. + z: The input skip tensor of shape + `(minibatch, n_pooled_feats, ..., n_ray_pts, skip_dim)` + which is appended to layers whose indices are specified by `input_skips`. + Returns: + y: The output tensor of shape + `(minibatch, 1, ..., n_ray_pts, input_dim)`. + """ + + if z is None: + # if the skip tensor is None, we use `x` instead. + z = x + + y = self.first(x) + + B, n_pool, n_rays, n_pts, dim = y.shape + + # y_p in n_pool, n_pts, B x n_rays x dim + y_p = y.permute(1, 3, 0, 2, 4) + + skipi = 0 + dimh = dim + for li, (layer_pool, layer_ray) in enumerate( + zip(self.layers_pool, self.layers_ray) + ): + y_pool_attn = y_p.reshape(n_pool, n_pts * B * n_rays, dimh) + if li in self._input_skips: + z_skip = self.skip_linear[skipi](z) + y_pool_attn = y_pool_attn + z_skip.permute(1, 3, 0, 2, 4).reshape( + n_pool, n_pts * B * n_rays, dimh + ) + skipi += 1 + # n_pool x B*n_rays*n_pts x dim + y_pool_attn, pool_attn = layer_pool(y_pool_attn, src_key_padding_mask=None) + dimh = y_pool_attn.shape[-1] + + y_ray_attn = ( + y_pool_attn.view(n_pool, n_pts, B * n_rays, dimh) + .permute(1, 0, 2, 3) + .reshape(n_pts, n_pool * B * n_rays, dimh) + ) + # n_pts x n_pool*B*n_rays x dim + y_ray_attn, ray_attn = layer_ray( + y_ray_attn, + src_key_padding_mask=None, + ) + + y_p = y_ray_attn.view(n_pts, n_pool, B * n_rays, dimh).permute(1, 0, 2, 3) + + y = y_p.view(n_pool, n_pts, B, n_rays, dimh).permute(2, 0, 3, 1, 4) + + W = torch.softmax(y[..., :1], dim=1) + y = (y * W).sum(dim=1) + y = self.last(y) + + return y + + +class TransformerEncoderLayer(torch.nn.Module): + r"""TransformerEncoderLayer is made up of self-attn and feedforward network. + This standard encoder layer is based on the paper "Attention Is All You Need". + Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, + Lukasz Kaiser, and Illia Polosukhin. 2017. Attention is all you need. In Advances in + Neural Information Processing Systems, pages 6000-6010. Users may modify or implement + in a different way during application. + + Args: + d_model: the number of expected features in the input (required). + nhead: the number of heads in the multiheadattention models (required). + dim_feedforward: the dimension of the feedforward network model (default=2048). + dropout: the dropout value (default=0.1). + activation: the activation function of intermediate layer, relu or gelu (default=relu). + + Examples:: + >>> encoder_layer = nn.TransformerEncoderLayer(d_model=512, nhead=8) + >>> src = torch.rand(10, 32, 512) + >>> out = encoder_layer(src) + """ + + def __init__( + self, d_model, nhead, dim_feedforward=2048, dropout=0.1, d_model_out=-1 + ): + super(TransformerEncoderLayer, self).__init__() + self.self_attn = torch.nn.MultiheadAttention(d_model, nhead, dropout=dropout) + # Implementation of Feedforward model + self.linear1 = torch.nn.Linear(d_model, dim_feedforward) + self.dropout = torch.nn.Dropout(dropout) + d_model_out = d_model if d_model_out <= 0 else d_model_out + self.linear2 = torch.nn.Linear(dim_feedforward, d_model_out) + self.norm1 = torch.nn.LayerNorm(d_model) + self.norm2 = torch.nn.LayerNorm(d_model_out) + self.dropout1 = torch.nn.Dropout(dropout) + self.dropout2 = torch.nn.Dropout(dropout) + + self.activation = torch.nn.functional.relu + + def forward(self, src, src_mask=None, src_key_padding_mask=None): + r"""Pass the input through the encoder layer. + + Args: + src: the sequence to the encoder layer (required). + src_mask: the mask for the src sequence (optional). + src_key_padding_mask: the mask for the src keys per batch (optional). + + Shape: + see the docs in Transformer class. + """ + src2, attn = self.self_attn( + src, src, src, attn_mask=src_mask, key_padding_mask=src_key_padding_mask + ) + src = src + self.dropout1(src2) + src = self.norm1(src) + src2 = self.linear2(self.dropout(self.activation(self.linear1(src)))) + d_out = src2.shape[-1] + src = src[..., :d_out] + self.dropout2(src2)[..., :d_out] + src = self.norm2(src) + return src, attn + + +def _xavier_init(linear) -> None: + """ + Performs the Xavier weight initialization of the linear layer `linear`. + """ + torch.nn.init.xavier_uniform_(linear.weight.data) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/idr_feature_field.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/idr_feature_field.py new file mode 100644 index 0000000000000000000000000000000000000000..be98b878489872b3e43a69b1956f3e59f02e1f03 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/idr_feature_field.py @@ -0,0 +1,176 @@ +# @lint-ignore-every LICENSELINT +# Adapted from https://github.com/lioryariv/idr/blob/main/code/model/ +# implicit_differentiable_renderer.py +# Copyright (c) 2020 Lior Yariv + +# pyre-unsafe +import math +from typing import Optional, Tuple + +import torch +from pytorch3d.implicitron.models.renderer.base import ImplicitronRayBundle +from pytorch3d.implicitron.tools.config import registry +from pytorch3d.renderer.implicit import HarmonicEmbedding + +from torch import nn + +from .base import ImplicitFunctionBase +from .utils import get_rays_points_world + + +@registry.register +class IdrFeatureField(ImplicitFunctionBase, torch.nn.Module): + """ + Implicit function as used in http://github.com/lioryariv/idr. + + Members: + d_in: dimension of the input point. + n_harmonic_functions_xyz: If -1, do not embed the point. + If >=0, use a harmonic embedding with this number of + harmonic functions. (The harmonic embedding includes the input + itself, so a value of 0 means the point is used but without + any harmonic functions.) + d_out and feature_vector_size: Sum of these is the output + dimension. This implicit function thus returns a concatenation + of `d_out` signed distance function values and `feature_vector_size` + features (such as colors). When used in `GenericModel`, + `feature_vector_size` corresponds is automatically set to + `render_features_dimensions`. + dims: list of hidden layer sizes. + geometric_init: whether to use custom weight initialization + in linear layers. If False, pytorch default (uniform sampling) + is used. + bias: if geometric_init=True, initial value for bias subtracted + in the last layer. + skip_in: List of indices of layers that receive as input the initial + value concatenated with the output of the previous layers. + weight_norm: whether to apply weight normalization to each layer. + pooled_feature_dim: If view pooling is in use (provided as + fun_viewpool to forward()) this must be its number of features. + Otherwise this must be set to 0. (If used from GenericModel, + this config value will be overridden automatically.) + encoding_dim: If global coding is in use (provided as global_code + to forward()) this must be its number of featuress. + Otherwise this must be set to 0. (If used from GenericModel, + this config value will be overridden automatically.) + """ + + feature_vector_size: int = 3 + d_in: int = 3 + d_out: int = 1 + dims: Tuple[int, ...] = (512, 512, 512, 512, 512, 512, 512, 512) + geometric_init: bool = True + bias: float = 1.0 + skip_in: Tuple[int, ...] = () + weight_norm: bool = True + n_harmonic_functions_xyz: int = 0 + pooled_feature_dim: int = 0 + encoding_dim: int = 0 + + def __post_init__(self): + dims = [self.d_in] + list(self.dims) + [self.d_out + self.feature_vector_size] + + self.embed_fn = None + if self.n_harmonic_functions_xyz >= 0: + self.embed_fn = HarmonicEmbedding( + self.n_harmonic_functions_xyz, append_input=True + ) + dims[0] = self.embed_fn.get_output_dim() + if self.pooled_feature_dim > 0: + dims[0] += self.pooled_feature_dim + if self.encoding_dim > 0: + dims[0] += self.encoding_dim + + self.num_layers = len(dims) + + out_dim = 0 + layers = [] + for layer_idx in range(self.num_layers - 1): + if layer_idx + 1 in self.skip_in: + out_dim = dims[layer_idx + 1] - dims[0] + else: + out_dim = dims[layer_idx + 1] + + lin = nn.Linear(dims[layer_idx], out_dim) + + if self.geometric_init: + if layer_idx == self.num_layers - 2: + torch.nn.init.normal_( + lin.weight, + mean=math.pi**0.5 / dims[layer_idx] ** 0.5, + std=0.0001, + ) + torch.nn.init.constant_(lin.bias, -self.bias) + elif self.n_harmonic_functions_xyz >= 0 and layer_idx == 0: + torch.nn.init.constant_(lin.bias, 0.0) + torch.nn.init.constant_(lin.weight[:, 3:], 0.0) + torch.nn.init.normal_(lin.weight[:, :3], 0.0, 2**0.5 / out_dim**0.5) + elif self.n_harmonic_functions_xyz >= 0 and layer_idx in self.skip_in: + torch.nn.init.constant_(lin.bias, 0.0) + torch.nn.init.normal_(lin.weight, 0.0, 2**0.5 / out_dim**0.5) + torch.nn.init.constant_(lin.weight[:, -(dims[0] - 3) :], 0.0) + else: + torch.nn.init.constant_(lin.bias, 0.0) + torch.nn.init.normal_(lin.weight, 0.0, 2**0.5 / out_dim**0.5) + + if self.weight_norm: + lin = nn.utils.weight_norm(lin) + + layers.append(lin) + + self.linear_layers = torch.nn.ModuleList(layers) + self.out_dim = out_dim + self.softplus = nn.Softplus(beta=100) + + # pyre-fixme[14]: `forward` overrides method defined in `ImplicitFunctionBase` + # inconsistently. + def forward( + self, + *, + ray_bundle: Optional[ImplicitronRayBundle] = None, + rays_points_world: Optional[torch.Tensor] = None, + fun_viewpool=None, + global_code=None, + **kwargs, + ): + # this field only uses point locations + # rays_points_world.shape = [minibatch x ... x pts_per_ray x 3] + rays_points_world = get_rays_points_world(ray_bundle, rays_points_world) + + if rays_points_world.numel() == 0 or ( + self.embed_fn is None and fun_viewpool is None and global_code is None + ): + return torch.tensor( + [], device=rays_points_world.device, dtype=rays_points_world.dtype + ).view(0, self.out_dim) + + embeddings = [] + if self.embed_fn is not None: + embeddings.append(self.embed_fn(rays_points_world)) + + if fun_viewpool is not None: + assert rays_points_world.ndim == 2 + pooled_feature = fun_viewpool(rays_points_world[None]) + # TODO: pooled features are 4D! + embeddings.append(pooled_feature) + + if global_code is not None: + assert global_code.shape[0] == 1 # TODO: generalize to batches! + # This will require changing raytracer code + # embedding = embedding[None].expand(global_code.shape[0], *embedding.shape) + embeddings.append( + global_code[0, None, :].expand(rays_points_world.shape[0], -1) + ) + + embedding = torch.cat(embeddings, dim=-1) + x = embedding + for layer_idx in range(self.num_layers - 1): + if layer_idx in self.skip_in: + x = torch.cat([x, embedding], dim=-1) / 2**0.5 + + x = self.linear_layers[layer_idx](x) + + if layer_idx < self.num_layers - 2: + x = self.softplus(x) + + return x diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/neural_radiance_field.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/neural_radiance_field.py new file mode 100644 index 0000000000000000000000000000000000000000..00f3aa65f2ef42012071cc85bffeaf39e241bb2b --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/neural_radiance_field.py @@ -0,0 +1,275 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import logging +from typing import Optional, Tuple + +import torch +from pytorch3d.common.linear_with_repeat import LinearWithRepeat +from pytorch3d.implicitron.models.renderer.base import ( + conical_frustum_to_gaussian, + ImplicitronRayBundle, +) +from pytorch3d.implicitron.tools.config import expand_args_fields, registry +from pytorch3d.renderer.cameras import CamerasBase +from pytorch3d.renderer.implicit import HarmonicEmbedding +from pytorch3d.renderer.implicit.utils import ray_bundle_to_ray_points + +from .base import ImplicitFunctionBase + +from .decoding_functions import ( # noqa + _xavier_init, + MLPWithInputSkips, + TransformerWithInputSkips, +) +from .utils import create_embeddings_for_implicit_function + + +logger = logging.getLogger(__name__) + + +class NeuralRadianceFieldBase(ImplicitFunctionBase, torch.nn.Module): + n_harmonic_functions_xyz: int = 10 + n_harmonic_functions_dir: int = 4 + n_hidden_neurons_dir: int = 128 + latent_dim: int = 0 + input_xyz: bool = True + xyz_ray_dir_in_camera_coords: bool = False + color_dim: int = 3 + use_integrated_positional_encoding: bool = False + """ + Args: + n_harmonic_functions_xyz: The number of harmonic functions + used to form the harmonic embedding of 3D point locations. + n_harmonic_functions_dir: The number of harmonic functions + used to form the harmonic embedding of the ray directions. + n_hidden_neurons_xyz: The number of hidden units in the + fully connected layers of the MLP that accepts the 3D point + locations and outputs the occupancy field with the intermediate + features. + n_hidden_neurons_dir: The number of hidden units in the + fully connected layers of the MLP that accepts the intermediate + features and ray directions and outputs the radiance field + (per-point colors). + n_layers_xyz: The number of layers of the MLP that outputs the + occupancy field. + append_xyz: The list of indices of the skip layers of the occupancy MLP. + use_integrated_positional_encoding: If True, use integrated positional enoding + as defined in `MIP-NeRF `_. + If False, use the classical harmonic embedding + defined in `NeRF `_. + """ + + def __post_init__(self): + # The harmonic embedding layer converts input 3D coordinates + # to a representation that is more suitable for + # processing with a deep neural network. + self.harmonic_embedding_xyz = HarmonicEmbedding( + self.n_harmonic_functions_xyz, append_input=True + ) + self.harmonic_embedding_dir = HarmonicEmbedding( + self.n_harmonic_functions_dir, append_input=True + ) + if not self.input_xyz and self.latent_dim <= 0: + raise ValueError("The latent dimension has to be > 0 if xyz is not input!") + + embedding_dim_dir = self.harmonic_embedding_dir.get_output_dim() + + self.xyz_encoder = self._construct_xyz_encoder( + input_dim=self.get_xyz_embedding_dim() + ) + + self.intermediate_linear = torch.nn.Linear( + self.n_hidden_neurons_xyz, self.n_hidden_neurons_xyz + ) + _xavier_init(self.intermediate_linear) + + self.density_layer = torch.nn.Linear(self.n_hidden_neurons_xyz, 1) + _xavier_init(self.density_layer) + + # Zero the bias of the density layer to avoid + # a completely transparent initialization. + self.density_layer.bias.data[:] = 0.0 # fixme: Sometimes this is not enough + + self.color_layer = torch.nn.Sequential( + LinearWithRepeat( + self.n_hidden_neurons_xyz + embedding_dim_dir, self.n_hidden_neurons_dir + ), + torch.nn.ReLU(True), + torch.nn.Linear(self.n_hidden_neurons_dir, self.color_dim), + torch.nn.Sigmoid(), + ) + + def get_xyz_embedding_dim(self): + return ( + self.harmonic_embedding_xyz.get_output_dim() * int(self.input_xyz) + + self.latent_dim + ) + + def _construct_xyz_encoder(self, input_dim: int): + raise NotImplementedError() + + def _get_colors(self, features: torch.Tensor, rays_directions: torch.Tensor): + """ + This function takes per-point `features` predicted by `self.xyz_encoder` + and evaluates the color model in order to attach to each + point a 3D vector of its RGB color. + """ + # Normalize the ray_directions to unit l2 norm. + rays_directions_normed = torch.nn.functional.normalize(rays_directions, dim=-1) + # Obtain the harmonic embedding of the normalized ray directions. + rays_embedding = self.harmonic_embedding_dir(rays_directions_normed) + + return self.color_layer((self.intermediate_linear(features), rays_embedding)) + + @staticmethod + def allows_multiple_passes() -> bool: + """ + Returns True as this implicit function allows + multiple passes. Overridden from ImplicitFunctionBase. + """ + return True + + def forward( + self, + *, + ray_bundle: ImplicitronRayBundle, + fun_viewpool=None, + camera: Optional[CamerasBase] = None, + global_code=None, + **kwargs, + ): + """ + The forward function accepts the parametrizations of + 3D points sampled along projection rays. The forward + pass is responsible for attaching a 3D vector + and a 1D scalar representing the point's + RGB color and opacity respectively. + + Args: + ray_bundle: An ImplicitronRayBundle object containing the following variables: + origins: A tensor of shape `(minibatch, ..., 3)` denoting the + origins of the sampling rays in world coords. + directions: A tensor of shape `(minibatch, ..., 3)` + containing the direction vectors of sampling rays in world coords. + lengths: A tensor of shape `(minibatch, ..., num_points_per_ray)` + containing the lengths at which the rays are sampled. + bins: An optional tensor of shape `(minibatch,..., num_points_per_ray + 1)` + containing the bins at which the rays are sampled. In this case + lengths is equal to the midpoints of bins. + + fun_viewpool: an optional callback with the signature + fun_fiewpool(points) -> pooled_features + where points is a [N_TGT x N x 3] tensor of world coords, + and pooled_features is a [N_TGT x ... x N_SRC x latent_dim] tensor + of the features pooled from the context images. + + Returns: + rays_densities: A tensor of shape `(minibatch, ..., num_points_per_ray, 1)` + denoting the opacitiy of each ray point. + rays_colors: A tensor of shape `(minibatch, ..., num_points_per_ray, 3)` + denoting the color of each ray point. + + Raises: + ValueError: If `use_integrated_positional_encoding` is True and + `ray_bundle.bins` is None. + """ + if self.use_integrated_positional_encoding and ray_bundle.bins is None: + raise ValueError( + "When use_integrated_positional_encoding is True, ray_bundle.bins must be set." + "Have you set to True `AbstractMaskRaySampler.use_bins_for_ray_sampling`?" + ) + + rays_points_world, diag_cov = ( + conical_frustum_to_gaussian(ray_bundle) + if self.use_integrated_positional_encoding + else (ray_bundle_to_ray_points(ray_bundle), None) # pyre-ignore + ) + # rays_points_world.shape = [minibatch x ... x pts_per_ray x 3] + + embeds = create_embeddings_for_implicit_function( + xyz_world=rays_points_world, + # for 2nd param but got `Union[None, torch.Tensor, torch.nn.Module]`. + xyz_embedding_function=( + self.harmonic_embedding_xyz if self.input_xyz else None + ), + global_code=global_code, + fun_viewpool=fun_viewpool, + xyz_in_camera_coords=self.xyz_ray_dir_in_camera_coords, + camera=camera, + diag_cov=diag_cov, + ) + + # embeds.shape = [minibatch x n_src x n_rays x n_pts x self.n_harmonic_functions*6+3] + features = self.xyz_encoder(embeds) + # features.shape = [minibatch x ... x self.n_hidden_neurons_xyz] + # NNs operate on the flattenned rays; reshaping to the correct spatial size + # TODO: maybe make the transformer work on non-flattened tensors to avoid this reshape + features = features.reshape(*rays_points_world.shape[:-1], -1) + + raw_densities = self.density_layer(features) + # raw_densities.shape = [minibatch x ... x 1] in [0-1] + + if self.xyz_ray_dir_in_camera_coords: + if camera is None: + raise ValueError("Camera must be given if xyz_ray_dir_in_camera_coords") + + directions = ray_bundle.directions @ camera.R + else: + directions = ray_bundle.directions + + rays_colors = self._get_colors(features, directions) + # rays_colors.shape = [minibatch x ... x 3] in [0-1] + + return raw_densities, rays_colors, {} + + +@registry.register +class NeuralRadianceFieldImplicitFunction(NeuralRadianceFieldBase): + transformer_dim_down_factor: float = 1.0 + n_hidden_neurons_xyz: int = 256 + n_layers_xyz: int = 8 + append_xyz: Tuple[int, ...] = (5,) + + def _construct_xyz_encoder(self, input_dim: int): + expand_args_fields(MLPWithInputSkips) + return MLPWithInputSkips( + self.n_layers_xyz, + input_dim, + self.n_hidden_neurons_xyz, + input_dim, + self.n_hidden_neurons_xyz, + input_skips=self.append_xyz, + ) + + +@registry.register +class NeRFormerImplicitFunction(NeuralRadianceFieldBase): + transformer_dim_down_factor: float = 2.0 + n_hidden_neurons_xyz: int = 80 + n_layers_xyz: int = 2 + append_xyz: Tuple[int, ...] = (1,) + + def _construct_xyz_encoder(self, input_dim: int): + return TransformerWithInputSkips( + self.n_layers_xyz, + input_dim, + self.n_hidden_neurons_xyz, + input_dim, + self.n_hidden_neurons_xyz, + input_skips=self.append_xyz, + dim_down_factor=self.transformer_dim_down_factor, + ) + + @staticmethod + def requires_pooling_without_aggregation() -> bool: + """ + Returns True as this implicit function needs + pooling without aggregation. Overridden from ImplicitFunctionBase. + """ + return True diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/scene_representation_networks.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/scene_representation_networks.py new file mode 100644 index 0000000000000000000000000000000000000000..6a605f06dd9ab136704d679b49782de68bdca38a --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/scene_representation_networks.py @@ -0,0 +1,429 @@ +# @lint-ignore-every LICENSELINT +# Adapted from https://github.com/vsitzmann/scene-representation-networks +# Copyright (c) 2019 Vincent Sitzmann + +# pyre-unsafe +from typing import Any, cast, Optional, Tuple + +import torch +from omegaconf import DictConfig +from pytorch3d.common.linear_with_repeat import LinearWithRepeat +from pytorch3d.implicitron.models.renderer.base import ImplicitronRayBundle +from pytorch3d.implicitron.third_party import hyperlayers, pytorch_prototyping +from pytorch3d.implicitron.tools.config import Configurable, registry, run_auto_creation +from pytorch3d.renderer import ray_bundle_to_ray_points +from pytorch3d.renderer.cameras import CamerasBase +from pytorch3d.renderer.implicit import HarmonicEmbedding + +from .base import ImplicitFunctionBase +from .utils import create_embeddings_for_implicit_function + + +def _kaiming_normal_init(module: torch.nn.Module) -> None: + if isinstance(module, (torch.nn.Linear, LinearWithRepeat)): + torch.nn.init.kaiming_normal_( + module.weight, a=0.0, nonlinearity="relu", mode="fan_in" + ) + + +class SRNRaymarchFunction(Configurable, torch.nn.Module): + n_harmonic_functions: int = 3 # 0 means raw 3D coord inputs + n_hidden_units: int = 256 + n_layers: int = 2 + in_features: int = 3 + out_features: int = 256 + latent_dim: int = 0 + xyz_in_camera_coords: bool = False + + # The internal network can be set as an output of an SRNHyperNet. + # Note that, in order to avoid Pytorch's automatic registering of the + # raymarch_function module on construction, we input the network wrapped + # as a 1-tuple. + + # raymarch_function should ideally be typed as Optional[Tuple[Callable]] + # but Omegaconf.structured doesn't like that. TODO: revisit after new + # release of omegaconf including https://github.com/omry/omegaconf/pull/749 . + raymarch_function: Any = None + + def __post_init__(self): + self._harmonic_embedding = HarmonicEmbedding( + self.n_harmonic_functions, append_input=True + ) + input_embedding_dim = ( + HarmonicEmbedding.get_output_dim_static( + self.in_features, + self.n_harmonic_functions, + True, + ) + + self.latent_dim + ) + + if self.raymarch_function is not None: + self._net = self.raymarch_function[0] + else: + self._net = pytorch_prototyping.FCBlock( + hidden_ch=self.n_hidden_units, + num_hidden_layers=self.n_layers, + in_features=input_embedding_dim, + out_features=self.out_features, + ) + + def forward( + self, + ray_bundle: ImplicitronRayBundle, + fun_viewpool=None, + camera: Optional[CamerasBase] = None, + global_code=None, + **kwargs, + ): + """ + Args: + ray_bundle: An ImplicitronRayBundle object containing the following variables: + origins: A tensor of shape `(minibatch, ..., 3)` denoting the + origins of the sampling rays in world coords. + directions: A tensor of shape `(minibatch, ..., 3)` + containing the direction vectors of sampling rays in world coords. + lengths: A tensor of shape `(minibatch, ..., num_points_per_ray)` + containing the lengths at which the rays are sampled. + fun_viewpool: an optional callback with the signature + fun_fiewpool(points) -> pooled_features + where points is a [N_TGT x N x 3] tensor of world coords, + and pooled_features is a [N_TGT x ... x N_SRC x latent_dim] tensor + of the features pooled from the context images. + + Returns: + rays_densities: A tensor of shape `(minibatch, ..., num_points_per_ray, 1)` + denoting the opacitiy of each ray point. + rays_colors: Set to None. + """ + # We first convert the ray parametrizations to world + # coordinates with `ray_bundle_to_ray_points`. + # pyre-ignore[6] + rays_points_world = ray_bundle_to_ray_points(ray_bundle) + + embeds = create_embeddings_for_implicit_function( + xyz_world=rays_points_world, + xyz_embedding_function=self._harmonic_embedding, + global_code=global_code, + fun_viewpool=fun_viewpool, + xyz_in_camera_coords=self.xyz_in_camera_coords, + camera=camera, + ) + + # Before running the network, we have to resize embeds to ndims=3, + # otherwise the SRN layers consume huge amounts of memory. + raymarch_features = self._net( + embeds.view(embeds.shape[0], -1, embeds.shape[-1]) + ) + # raymarch_features.shape = [minibatch x ... x self.n_hidden_neurons_xyz] + + # NNs operate on the flattenned rays; reshaping to the correct spatial size + raymarch_features = raymarch_features.reshape(*rays_points_world.shape[:-1], -1) + + return raymarch_features, None + + +class SRNPixelGenerator(Configurable, torch.nn.Module): + n_harmonic_functions: int = 4 + n_hidden_units: int = 256 + n_hidden_units_color: int = 128 + n_layers: int = 2 + in_features: int = 256 + out_features: int = 3 + ray_dir_in_camera_coords: bool = False + + def __post_init__(self): + self._harmonic_embedding = HarmonicEmbedding( + self.n_harmonic_functions, append_input=True + ) + self._net = pytorch_prototyping.FCBlock( + hidden_ch=self.n_hidden_units, + num_hidden_layers=self.n_layers, + in_features=self.in_features, + out_features=self.n_hidden_units, + ) + self._density_layer = torch.nn.Linear(self.n_hidden_units, 1) + self._density_layer.apply(_kaiming_normal_init) + embedding_dim_dir = self._harmonic_embedding.get_output_dim(input_dims=3) + self._color_layer = torch.nn.Sequential( + LinearWithRepeat( + self.n_hidden_units + embedding_dim_dir, + self.n_hidden_units_color, + ), + torch.nn.LayerNorm([self.n_hidden_units_color]), + torch.nn.ReLU(inplace=True), + torch.nn.Linear(self.n_hidden_units_color, self.out_features), + ) + self._color_layer.apply(_kaiming_normal_init) + + # TODO: merge with NeuralRadianceFieldBase's _get_colors + def _get_colors(self, features: torch.Tensor, rays_directions: torch.Tensor): + """ + This function takes per-point `features` predicted by `self.net` + and evaluates the color model in order to attach to each + point a 3D vector of its RGB color. + """ + # Normalize the ray_directions to unit l2 norm. + rays_directions_normed = torch.nn.functional.normalize(rays_directions, dim=-1) + # Obtain the harmonic embedding of the normalized ray directions. + rays_embedding = self._harmonic_embedding(rays_directions_normed) + return self._color_layer((features, rays_embedding)) + + def forward( + self, + raymarch_features: torch.Tensor, + ray_bundle: ImplicitronRayBundle, + camera: Optional[CamerasBase] = None, + **kwargs, + ): + """ + Args: + raymarch_features: Features from the raymarching network of shape + `(minibatch, ..., self.in_features)` + ray_bundle: An ImplicitronRayBundle object containing the following variables: + origins: A tensor of shape `(minibatch, ..., 3)` denoting the + origins of the sampling rays in world coords. + directions: A tensor of shape `(minibatch, ..., 3)` + containing the direction vectors of sampling rays in world coords. + lengths: A tensor of shape `(minibatch, ..., num_points_per_ray)` + containing the lengths at which the rays are sampled. + + Returns: + rays_densities: A tensor of shape `(minibatch, ..., num_points_per_ray, 1)` + denoting the opacitiy of each ray point. + rays_colors: A tensor of shape `(minibatch, ..., num_points_per_ray, 3)` + denoting the color of each ray point. + """ + # raymarch_features.shape = [minibatch x ... x pts_per_ray x 3] + features = self._net(raymarch_features) + # features.shape = [minibatch x ... x self.n_hidden_units] + + if self.ray_dir_in_camera_coords: + if camera is None: + raise ValueError("Camera must be given if xyz_ray_dir_in_camera_coords") + + directions = ray_bundle.directions @ camera.R + else: + directions = ray_bundle.directions + + # NNs operate on the flattenned rays; reshaping to the correct spatial size + features = features.reshape(*raymarch_features.shape[:-1], -1) + + raw_densities = self._density_layer(features) + + rays_colors = self._get_colors(features, directions) + + return raw_densities, rays_colors + + +class SRNRaymarchHyperNet(Configurable, torch.nn.Module): + """ + This is a raymarching function which has a forward like SRNRaymarchFunction + but instead of the weights being parameters of the module, they + are the output of another network, the hypernet, which takes the global_code + as input. All the dataclass members of SRNRaymarchFunction are here with the + same meaning. In addition, there are members with names ending `_hypernet` + which affect the hypernet. + + Because this class may be called repeatedly for the same global_code, the + output of the hypernet is cached in self.cached_srn_raymarch_function. + This member must be manually set to None whenever the global_code changes. + """ + + n_harmonic_functions: int = 3 # 0 means raw 3D coord inputs + n_hidden_units: int = 256 + n_layers: int = 2 + n_hidden_units_hypernet: int = 256 + n_layers_hypernet: int = 1 + in_features: int = 3 + out_features: int = 256 + latent_dim_hypernet: int = 0 + latent_dim: int = 0 + xyz_in_camera_coords: bool = False + + def __post_init__(self): + raymarch_input_embedding_dim = ( + HarmonicEmbedding.get_output_dim_static( + self.in_features, + self.n_harmonic_functions, + True, + ) + + self.latent_dim + ) + + self._hypernet = hyperlayers.HyperFC( + hyper_in_ch=self.latent_dim_hypernet, + hyper_num_hidden_layers=self.n_layers_hypernet, + hyper_hidden_ch=self.n_hidden_units_hypernet, + hidden_ch=self.n_hidden_units, + num_hidden_layers=self.n_layers, + in_ch=raymarch_input_embedding_dim, + out_ch=self.n_hidden_units, + ) + + self.cached_srn_raymarch_function: Optional[Tuple[SRNRaymarchFunction]] = None + + def _run_hypernet(self, global_code: torch.Tensor) -> Tuple[SRNRaymarchFunction]: + """ + Runs the hypernet and returns a 1-tuple containing the generated + srn_raymarch_function. + """ + + net = self._hypernet(global_code) + + # use the hyper-net generated network to instantiate the raymarch module + srn_raymarch_function = SRNRaymarchFunction( + n_harmonic_functions=self.n_harmonic_functions, + n_hidden_units=self.n_hidden_units, + n_layers=self.n_layers, + in_features=self.in_features, + out_features=self.out_features, + latent_dim=self.latent_dim, + xyz_in_camera_coords=self.xyz_in_camera_coords, + raymarch_function=(net,), + ) + + # move the generated raymarch function to the correct device + srn_raymarch_function.to(global_code.device) + + return (srn_raymarch_function,) + + def forward( + self, + ray_bundle: ImplicitronRayBundle, + fun_viewpool=None, + camera: Optional[CamerasBase] = None, + global_code=None, + **kwargs, + ): + + if global_code is None: + raise ValueError("SRN Hypernetwork requires a non-trivial global code.") + + # The raymarching network is cached in case the function is called repeatedly + # across LSTM iterations for the same global_code. + if self.cached_srn_raymarch_function is None: + # generate the raymarching network from the hypernet + self.cached_srn_raymarch_function = self._run_hypernet(global_code) + (srn_raymarch_function,) = cast( + Tuple[SRNRaymarchFunction], self.cached_srn_raymarch_function + ) + + return srn_raymarch_function( + ray_bundle=ray_bundle, + fun_viewpool=fun_viewpool, + camera=camera, + global_code=None, # the hypernetwork takes the global code + ) + + +@registry.register +# pyre-fixme[13]: Uninitialized attribute +class SRNImplicitFunction(ImplicitFunctionBase, torch.nn.Module): + latent_dim: int = 0 + raymarch_function: SRNRaymarchFunction + pixel_generator: SRNPixelGenerator + + def __post_init__(self): + run_auto_creation(self) + + def create_raymarch_function(self) -> None: + self.raymarch_function = SRNRaymarchFunction( + latent_dim=self.latent_dim, + **self.raymarch_function_args, + ) + + @classmethod + def raymarch_function_tweak_args(cls, type, args: DictConfig) -> None: + args.pop("latent_dim", None) + + def forward( + self, + *, + ray_bundle: ImplicitronRayBundle, + fun_viewpool=None, + camera: Optional[CamerasBase] = None, + global_code=None, + raymarch_features: Optional[torch.Tensor] = None, + **kwargs, + ): + predict_colors = raymarch_features is not None + if predict_colors: + return self.pixel_generator( + raymarch_features=raymarch_features, + ray_bundle=ray_bundle, + camera=camera, + **kwargs, + ) + else: + return self.raymarch_function( + ray_bundle=ray_bundle, + fun_viewpool=fun_viewpool, + camera=camera, + global_code=global_code, + **kwargs, + ) + + +@registry.register +# pyre-fixme[13]: Uninitialized attribute +class SRNHyperNetImplicitFunction(ImplicitFunctionBase, torch.nn.Module): + """ + This implicit function uses a hypernetwork to generate the + SRNRaymarchingFunction, and this is cached. Whenever the + global_code changes, `on_bind_args` must be called to clear + the cache. + """ + + latent_dim_hypernet: int = 0 + latent_dim: int = 0 + hypernet: SRNRaymarchHyperNet + pixel_generator: SRNPixelGenerator + + def __post_init__(self): + run_auto_creation(self) + + def create_hypernet(self) -> None: + self.hypernet = SRNRaymarchHyperNet( + latent_dim=self.latent_dim, + latent_dim_hypernet=self.latent_dim_hypernet, + **self.hypernet_args, + ) + + @classmethod + def hypernet_tweak_args(cls, type, args: DictConfig) -> None: + args.pop("latent_dim", None) + args.pop("latent_dim_hypernet", None) + + def forward( + self, + *, + ray_bundle: ImplicitronRayBundle, + fun_viewpool=None, + camera: Optional[CamerasBase] = None, + global_code=None, + raymarch_features: Optional[torch.Tensor] = None, + **kwargs, + ): + predict_colors = raymarch_features is not None + if predict_colors: + return self.pixel_generator( + raymarch_features=raymarch_features, + ray_bundle=ray_bundle, + camera=camera, + **kwargs, + ) + else: + return self.hypernet( + ray_bundle=ray_bundle, + fun_viewpool=fun_viewpool, + camera=camera, + global_code=global_code, + **kwargs, + ) + + def on_bind_args(self): + """ + The global_code may have changed, so we reset the hypernet. + """ + self.hypernet.cached_srn_raymarch_function = None diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/utils.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ec453f43e454c71df3cc5448d2c0ca961b59b7df --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/utils.py @@ -0,0 +1,221 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from typing import Callable, Optional + +import torch + +import torch.nn.functional as F +from pytorch3d.common.compat import prod +from pytorch3d.implicitron.models.renderer.base import ImplicitronRayBundle +from pytorch3d.renderer import ray_bundle_to_ray_points +from pytorch3d.renderer.cameras import CamerasBase + + +def broadcast_global_code(embeds: torch.Tensor, global_code: torch.Tensor): + """ + Expands the `global_code` of shape (minibatch, dim) + so that it can be appended to `embeds` of shape (minibatch, ..., dim2), + and appends to the last dimension of `embeds`. + """ + bs = embeds.shape[0] + global_code_broadcast = global_code.view(bs, *([1] * (embeds.ndim - 2)), -1).expand( + *embeds.shape[:-1], + global_code.shape[-1], + ) + return torch.cat([embeds, global_code_broadcast], dim=-1) + + +def create_embeddings_for_implicit_function( + xyz_world: torch.Tensor, + xyz_in_camera_coords: bool, + global_code: Optional[torch.Tensor], + camera: Optional[CamerasBase], + fun_viewpool: Optional[Callable], + xyz_embedding_function: Optional[Callable], + diag_cov: Optional[torch.Tensor] = None, +) -> torch.Tensor: + + bs, *spatial_size, pts_per_ray, _ = xyz_world.shape + + if xyz_in_camera_coords: + if camera is None: + raise ValueError("Camera must be given if xyz_in_camera_coords") + + ray_points_for_embed = ( + camera.get_world_to_view_transform() + .transform_points(xyz_world.view(bs, -1, 3)) + .view(xyz_world.shape) + ) + else: + ray_points_for_embed = xyz_world + + if xyz_embedding_function is None: + embeds = torch.empty( + bs, + 1, + prod(spatial_size), + pts_per_ray, + 0, + ) + else: + + embeds = xyz_embedding_function(ray_points_for_embed, diag_cov=diag_cov) + embeds = embeds.reshape( + bs, + 1, + prod(spatial_size), + pts_per_ray, + -1, + ) # flatten spatial, add n_src dim + + if fun_viewpool is not None: + # viewpooling + embeds_viewpooled = fun_viewpool(xyz_world.reshape(bs, -1, 3)) + embed_shape = ( + bs, + embeds_viewpooled.shape[1], + prod(spatial_size), + pts_per_ray, + -1, + ) + embeds_viewpooled = embeds_viewpooled.reshape(*embed_shape) + if embeds is not None: + embeds = torch.cat([embeds.expand(*embed_shape), embeds_viewpooled], dim=-1) + else: + embeds = embeds_viewpooled + + if global_code is not None: + # append the broadcasted global code to embeds + embeds = broadcast_global_code(embeds, global_code) + + return embeds + + +def interpolate_line( + points: torch.Tensor, + source: torch.Tensor, + **kwargs, +) -> torch.Tensor: + """ + Linearly interpolates values of source grids. The first dimension of points represents + number of points and the second coordinate, for example ([[x0], [x1], ...]). The first + dimension of argument source represents feature and ones after that the spatial + dimension. + + Arguments: + points: shape (n_grids, n_points, 1), + source: tensor of shape (n_grids, features, width), + Returns: + interpolated tensor of shape (n_grids, n_points, features) + """ + # To enable sampling of the source using the torch.functional.grid_sample + # points need to have 2 coordinates. + expansion = points.new_zeros(points.shape) + points = torch.cat((points, expansion), dim=-1) + + source = source[:, :, None, :] + points = points[:, :, None, :] + + out = F.grid_sample( + grid=points, + input=source, + **kwargs, + ) + return out[:, :, :, 0].permute(0, 2, 1) + + +def interpolate_plane( + points: torch.Tensor, + source: torch.Tensor, + **kwargs, +) -> torch.Tensor: + """ + Bilinearly interpolates values of source grids. The first dimension of points represents + number of points and the second coordinates, for example ([[x0, y0], [x1, y1], ...]). + The first dimension of argument source represents feature and ones after that the + spatial dimension. + + Arguments: + points: shape (n_grids, n_points, 2), + source: tensor of shape (n_grids, features, width, height), + Returns: + interpolated tensor of shape (n_grids, n_points, features) + """ + # permuting because torch.nn.functional.grid_sample works with + # (features, height, width) and not + # (features, width, height) + source = source.permute(0, 1, 3, 2) + points = points[:, :, None, :] + + out = F.grid_sample( + grid=points, + input=source, + **kwargs, + ) + return out[:, :, :, 0].permute(0, 2, 1) + + +def interpolate_volume( + points: torch.Tensor, source: torch.Tensor, **kwargs +) -> torch.Tensor: + """ + Interpolates values of source grids. The first dimension of points represents + number of points and the second coordinates, for example + [[x0, y0, z0], [x1, y1, z1], ...]. The first dimension of a source represents features + and ones after that the spatial dimension. + + Arguments: + points: shape (n_grids, n_points, 3), + source: tensor of shape (n_grids, features, width, height, depth), + Returns: + interpolated tensor of shape (n_grids, n_points, features) + """ + if "mode" in kwargs and kwargs["mode"] == "trilinear": + kwargs = kwargs.copy() + kwargs["mode"] = "bilinear" + # permuting because torch.nn.functional.grid_sample works with + # (features, depth, height, width) and not (features, width, height, depth) + source = source.permute(0, 1, 4, 3, 2) + grid = points[:, :, None, None, :] + + out = F.grid_sample( + grid=grid, + input=source, + **kwargs, + ) + return out[:, :, :, 0, 0].permute(0, 2, 1) + + +def get_rays_points_world( + ray_bundle: Optional[ImplicitronRayBundle] = None, + rays_points_world: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """ + Converts the ray_bundle to rays_points_world if rays_points_world is not defined + and raises error if both are defined. + + Args: + ray_bundle: An ImplicitronRayBundle object or None + rays_points_world: A torch.Tensor representing ray points converted to + world coordinates + Returns: + A torch.Tensor representing ray points converted to world coordinates + of shape [minibatch x ... x pts_per_ray x 3]. + """ + if rays_points_world is not None and ray_bundle is not None: + raise ValueError( + "Cannot define both rays_points_world and ray_bundle," + + " one has to be None." + ) + if rays_points_world is not None: + return rays_points_world + if ray_bundle is not None: + # pyre-ignore[6] + return ray_bundle_to_ray_points(ray_bundle) + raise ValueError("ray_bundle and rays_points_world cannot both be None") diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/voxel_grid.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/voxel_grid.py new file mode 100644 index 0000000000000000000000000000000000000000..0c106b2aa9bc1691c21a84d1960c5d0b8c11c67d --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/voxel_grid.py @@ -0,0 +1,1139 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +""" +This file contains classes that implement Voxel grids, both in their full resolution +as in the factorized form. There are two factorized forms implemented, Tensor rank decomposition +or CANDECOMP/PARAFAC (here CP) and Vector Matrix (here VM) factorization from the +TensoRF (https://arxiv.org/abs/2203.09517) paper. + +In addition, the module VoxelGridModule implements a trainable instance of one of +these classes. + +""" + +import logging +import warnings +from collections.abc import Mapping +from dataclasses import dataclass, field + +from distutils.version import LooseVersion +from typing import Any, Callable, ClassVar, Dict, Iterator, List, Optional, Tuple, Type + +import torch +from omegaconf import DictConfig +from pytorch3d.implicitron.tools.config import ( + Configurable, + registry, + ReplaceableBase, + run_auto_creation, +) +from pytorch3d.structures.volumes import VolumeLocator + +from .utils import interpolate_line, interpolate_plane, interpolate_volume + + +logger = logging.getLogger(__name__) + + +@dataclass +class VoxelGridValuesBase: + pass + + +class VoxelGridBase(ReplaceableBase, torch.nn.Module): + """ + Base class for all the voxel grid variants whith added trilinear interpolation between + voxels (for example if voxel (0.333, 1, 3) is queried that would return the result + 2/3*voxel[0, 1, 3] + 1/3*voxel[1, 1, 3]) + + Internally voxel grids are indexed by (features, x, y, z). If queried the point is not + inside the voxel grid the vector that will be returned is determined by padding. + + Members: + align_corners: parameter used in torch.functional.grid_sample. For details go to + https://pytorch.org/docs/stable/generated/torch.nn.functional.grid_sample.html by + default is True + padding: padding mode for outside grid values 'zeros' | 'border' | 'reflection'. + Default is 'zeros' + mode: interpolation mode to calculate output values : + 'bilinear' | 'nearest' | 'bicubic' | 'trilinear'. + Default: 'bilinear' Note: mode='bicubic' supports only FullResolutionVoxelGrid. + When mode='bilinear' and the input is 5-D, the interpolation mode used internally + will actually be trilinear. + n_features: number of dimensions of base feature vector. Determines how many features + the grid returns. + resolution_changes: a dictionary, where keys are change epochs and values are + 3-tuples containing x, y, z grid sizes corresponding to each axis to each epoch + """ + + align_corners: bool = True + padding: str = "zeros" + mode: str = "bilinear" + n_features: int = 1 + # return the line below once we drop OmegaConf 2.1 support + # resolution_changes: Dict[int, List[int]] = field( + resolution_changes: Dict[int, Any] = field( + default_factory=lambda: {0: [128, 128, 128]} + ) + + def __post_init__(self): + if 0 not in self.resolution_changes: + raise ValueError("There has to be key `0` in `resolution_changes`.") + + def evaluate_world( + self, + points: torch.Tensor, + grid_values: VoxelGridValuesBase, + locator: VolumeLocator, + ) -> torch.Tensor: + """ + Evaluates the voxel grid at points in the world coordinate frame. + The interpolation type is determined by the `mode` member. + + Arguments: + points (torch.Tensor): tensor of points that you want to query + of a form (n_grids, ..., 3) + grid_values: an object of type Class.values_type which has tensors as + members which have shapes derived from the get_shapes() method + locator: a VolumeLocator object + Returns: + torch.Tensor: shape (n_grids, ..., n_features) + """ + points_local = locator.world_to_local_coords(points) + return self.evaluate_local(points_local, grid_values) + + def evaluate_local( + self, points: torch.Tensor, grid_values: VoxelGridValuesBase + ) -> torch.Tensor: + """ + Evaluates the voxel grid at points in the local coordinate frame, + The interpolation type is determined by the `mode` member. + + Arguments: + points (torch.Tensor): tensor of points that you want to query + of a form (n_grids, ..., 3), in a normalized form (coordinates are in [-1, 1]) + grid_values: an object of type VMFactorizedVoxelGrid.values_type which has tensors + as members which have shapes derived from the get_shapes() method + Returns: + torch.Tensor: shape (n_grids, ..., n_features) + """ + raise NotImplementedError() + + def get_shapes(self, epoch: int) -> Dict[str, Tuple]: + """ + Using parameters from the __init__ method, this method returns the + shapes of individual tensors needed to run the evaluate method. + + Args: + epoch: If the shape varies during training, which training epoch's shape to return. + Returns: + a dictionary of needed shapes. To use the evaluate_local and evaluate_world methods + replace the shapes in the dictionary with tensors of those shapes and add the + first 'batch' dimension. If the required shape is (a, b) and you want to + have g grids then the tensor that replaces the shape should have the + shape (g, a, b). + """ + raise NotImplementedError() + + def get_resolution(self, epoch: int) -> List[int]: + """ + Returns the resolution which the grid should have at specific epoch + + Args: + epoch which to use in the resolution calculation + Returns: + resolution at specific epoch + """ + last_change = 0 + for change_epoch in self.resolution_changes: + if change_epoch <= epoch: + last_change = max(last_change, change_epoch) + return self.resolution_changes[last_change] + + @staticmethod + def get_output_dim(args: DictConfig) -> int: + """ + Given all the arguments of the grid's __init__, returns output's last dimension length. + + In particular, if self.evaluate_world or self.evaluate_local + are called with `points` of shape (n_grids, n_points, 3), + their output will be of shape + (n_grids, n_points, grid.get_output_dim()). + + Args: + args: DictConfig which would be used to initialize the object + Returns: + output's last dimension length + """ + return args["n_features"] + + def change_resolution( + self, + grid_values: VoxelGridValuesBase, + *, + epoch: Optional[int] = None, + grid_values_with_wanted_resolution: Optional[VoxelGridValuesBase] = None, + mode: str = "linear", + align_corners: bool = True, + antialias: bool = False, + ) -> Tuple[VoxelGridValuesBase, bool]: + """ + Changes resolution of tensors in `grid_values` to match the + `grid_values_with_wanted_resolution` or resolution on wanted epoch. + + Args: + epoch: current training epoch, used to see if the grid needs regridding + grid_values: instance of self.values_type which contains + the voxel grid which will be interpolated to create the new grid + epoch: epoch which is used to get the resolution of the new + `grid_values` using `self.resolution_changes`. + grid_values_with_wanted_resolution: `VoxelGridValuesBase` to whose resolution + to interpolate grid_values + align_corners: as for torch.nn.functional.interpolate + mode: as for torch.nn.functional.interpolate + 'nearest' | 'bicubic' | 'linear' | 'area' | 'nearest-exact'. + Default: 'linear' + antialias: as for torch.nn.functional.interpolate. + Using anti-alias option + together with align_corners=False and mode='bicubic', interpolation + result would match Pillow result for downsampling operation. + Supported mode: 'bicubic' + Returns: + tuple of + - new voxel grid_values of desired resolution, of type self.values_type + - True if regridding has happened. + """ + + if (epoch is None) == (grid_values_with_wanted_resolution is None): + raise ValueError( + "Exactly one of `epoch` or " + "`grid_values_with_wanted_resolution` has to be defined." + ) + + if mode not in ("nearest", "bicubic", "linear", "area", "nearest-exact"): + raise ValueError( + "`mode` should be one of the following 'nearest'" + + "| 'bicubic' | 'linear' | 'area' | 'nearest-exact'" + ) + + interpolate_has_antialias = LooseVersion(torch.__version__) >= "1.11" + + if antialias and not interpolate_has_antialias: + warnings.warn("Antialiased interpolation requires PyTorch 1.11+; ignoring") + + interp_kwargs = {"antialias": antialias} if interpolate_has_antialias else {} + + def change_individual_resolution(tensor, wanted_resolution): + if mode == "linear": + n_dim = len(wanted_resolution) + new_mode = ("linear", "bilinear", "trilinear")[n_dim - 1] + else: + new_mode = mode + return torch.nn.functional.interpolate( + input=tensor, + size=wanted_resolution, + mode=new_mode, + align_corners=align_corners, + recompute_scale_factor=False, + **interp_kwargs, + ) + + if epoch is not None: + if epoch not in self.resolution_changes: + return grid_values, False + + wanted_shapes = self.get_shapes(epoch=epoch) + params = { + name: change_individual_resolution( + getattr(grid_values, name), shape[1:] + ) + for name, shape in wanted_shapes.items() + } + res = self.get_resolution(epoch) + logger.info(f"Changed grid resolutiuon at epoch {epoch} to {res}") + else: + params = { + name: ( + change_individual_resolution( + getattr(grid_values, name), tensor.shape[2:] + ) + if tensor is not None + else None + ) + for name, tensor in vars(grid_values_with_wanted_resolution).items() + } + + return self.values_type(**params), True + + def get_resolution_change_epochs(self) -> Tuple[int, ...]: + """ + Returns epochs at which this grid should change epochs. + """ + return tuple(self.resolution_changes.keys()) + + def get_align_corners(self) -> bool: + """ + Returns True if voxel grid uses align_corners=True + """ + return self.align_corners + + def crop_world( + self, + min_point_world: torch.Tensor, + max_point_world: torch.Tensor, + grid_values: VoxelGridValuesBase, + volume_locator: VolumeLocator, + ) -> VoxelGridValuesBase: + """ + Crops the voxel grid based on minimum and maximum occupied point in + world coordinates. After cropping all 8 corner points are preserved in + the voxel grid. This is achieved by preserving all the voxels needed to + calculate the point. + + +--------B + / /| + / / | + +--------+ | <==== Bounding box represented by points A and B: + | | | - B has x, y and z coordinates bigger or equal + | | + to all other points of the object + | | / - A has x, y and z coordinates smaller or equal + | |/ to all other points of the object + A--------+ + + Args: + min_point_world: torch.Tensor of shape (3,). Has x, y and z coordinates + smaller or equal to all other occupied points. Point A from the + picture above. + max_point_world: torch.Tensor of shape (3,). Has x, y and z coordinates + bigger or equal to all other occupied points. Point B from the + picture above. + grid_values: instance of self.values_type which contains + the voxel grid which will be cropped to create the new grid + volume_locator: VolumeLocator object used to convert world to local + cordinates + Returns: + instance of self.values_type which has volume cropped to desired size. + """ + min_point_local = volume_locator.world_to_local_coords(min_point_world[None])[0] + max_point_local = volume_locator.world_to_local_coords(max_point_world[None])[0] + return self.crop_local(min_point_local, max_point_local, grid_values) + + def crop_local( + self, + min_point_local: torch.Tensor, + max_point_local: torch.Tensor, + grid_values: VoxelGridValuesBase, + ) -> VoxelGridValuesBase: + """ + Crops the voxel grid based on minimum and maximum occupied point in local + coordinates. After cropping both min and max point are preserved in the voxel + grid. This is achieved by preserving all the voxels needed to calculate the point. + + +--------B + / /| + / / | + +--------+ | <==== Bounding box represented by points A and B: + | | | - B has x, y and z coordinates bigger or equal + | | + to all other points of the object + | | / - A has x, y and z coordinates smaller or equal + | |/ to all other points of the object + A--------+ + + Args: + min_point_local: torch.Tensor of shape (3,). Has x, y and z coordinates + smaller or equal to all other occupied points. Point A from the + picture above. All elements in [-1, 1]. + max_point_local: torch.Tensor of shape (3,). Has x, y and z coordinates + bigger or equal to all other occupied points. Point B from the + picture above. All elements in [-1, 1]. + grid_values: instance of self.values_type which contains + the voxel grid which will be cropped to create the new grid + Returns: + instance of self.values_type which has volume cropped to desired size. + """ + raise NotImplementedError() + + +@dataclass +class FullResolutionVoxelGridValues(VoxelGridValuesBase): + voxel_grid: torch.Tensor + + +@registry.register +class FullResolutionVoxelGrid(VoxelGridBase): + """ + Full resolution voxel grid equivalent to 4D tensor where shape is + (features, width, height, depth) with linear interpolation between voxels. + """ + + # the type of grid_values argument needed to run evaluate_local() + values_type: ClassVar[Type[VoxelGridValuesBase]] = FullResolutionVoxelGridValues + + # pyre-fixme[14]: `evaluate_local` overrides method defined in `VoxelGridBase` + # inconsistently. + def evaluate_local( + self, points: torch.Tensor, grid_values: FullResolutionVoxelGridValues + ) -> torch.Tensor: + """ + Evaluates the voxel grid at points in the local coordinate frame, + The interpolation type is determined by the `mode` member. + + Arguments: + points (torch.Tensor): tensor of points that you want to query + of a form (..., 3), in a normalized form (coordinates are in [-1, 1]) + grid_values: an object of type values_type which has tensors as + members which have shapes derived from the get_shapes() method + Returns: + torch.Tensor: shape (n_grids, ..., n_features) + """ + # (n_grids, n_points_total, n_features) from (n_grids, ..., n_features) + recorded_shape = points.shape + points = points.view(points.shape[0], -1, points.shape[-1]) + interpolated = interpolate_volume( + points, + grid_values.voxel_grid, + align_corners=self.align_corners, + padding_mode=self.padding, + mode=self.mode, + ) + return interpolated.view(*recorded_shape[:-1], -1) + + def get_shapes(self, epoch: int) -> Dict[str, Tuple]: + width, height, depth = self.get_resolution(epoch) + return {"voxel_grid": (self.n_features, width, height, depth)} + + # pyre-ignore[14] + def crop_local( + self, + min_point_local: torch.Tensor, + max_point_local: torch.Tensor, + grid_values: FullResolutionVoxelGridValues, + ) -> FullResolutionVoxelGridValues: + assert torch.all(min_point_local < max_point_local) + min_point_local = torch.clamp(min_point_local, -1, 1) + max_point_local = torch.clamp(max_point_local, -1, 1) + _, _, width, height, depth = grid_values.voxel_grid.shape + resolution = grid_values.voxel_grid.new_tensor([width, height, depth]) + min_point_local01 = (min_point_local + 1) / 2 + max_point_local01 = (max_point_local + 1) / 2 + + if self.align_corners: + minx, miny, minz = torch.floor(min_point_local01 * (resolution - 1)).long() + maxx, maxy, maxz = torch.ceil(max_point_local01 * (resolution - 1)).long() + else: + minx, miny, minz = torch.floor(min_point_local01 * resolution - 0.5).long() + maxx, maxy, maxz = torch.ceil(max_point_local01 * resolution - 0.5).long() + + return FullResolutionVoxelGridValues( + voxel_grid=grid_values.voxel_grid[ + :, :, minx : maxx + 1, miny : maxy + 1, minz : maxz + 1 + ] + ) + + +@dataclass +class CPFactorizedVoxelGridValues(VoxelGridValuesBase): + vector_components_x: torch.Tensor + vector_components_y: torch.Tensor + vector_components_z: torch.Tensor + basis_matrix: Optional[torch.Tensor] = None + + +@registry.register +class CPFactorizedVoxelGrid(VoxelGridBase): + """ + Canonical Polyadic (CP/CANDECOMP/PARAFAC) Factorization factorizes the 3d grid into three + vectors (x, y, z). For n_components=n, the 3d grid is a sum of the two outer products + (call it ⊗) of each vector type (x, y, z): + + 3d_grid = x0 ⊗ y0 ⊗ z0 + x1 ⊗ y1 ⊗ z1 + ... + xn ⊗ yn ⊗ zn + + These tensors are passed in a object of CPFactorizedVoxelGridValues (here obj) as + obj.vector_components_x, obj.vector_components_y, obj.vector_components_z. Their shapes are + `(n_components, r)` where `r` is the relevant resolution. + + Each element of this sum has an extra dimension, which gets matrix-multiplied by an + appropriate "basis matrix" of shape (n_grids, n_components, n_features). This multiplication + brings us to the desired "n_features" dimensionality. If basis_matrix=False the elements + of different components are summed together to create (n_grids, n_components, 1) tensor. + With some notation abuse, ignoring the interpolation operation, simplifying and denoting + n_features as F, n_components as C and n_grids as G: + + 3d_grid = (x ⊗ y ⊗ z) @ basis # GWHDC x GCF -> GWHDF + + The basis feature vectors are passed as obj.basis_matrix. + + Members: + n_components: number of vector triplets, higher number gives better approximation. + basis_matrix: how to transform components. If matrix_reduction=True result + matrix of shape (n_grids, n_points_total, n_components) is batch matrix multiplied + by the basis_matrix of shape (n_grids, n_components, n_features). If + matrix_reduction=False, the result tensor of (n_grids, n_points_total, n_components) + is summed along the rows to get (n_grids, n_points_total, 1), which is then viewed + to return to starting shape (n_grids, ..., 1). + """ + + # the type of grid_values argument needed to run evaluate_local() + values_type: ClassVar[Type[VoxelGridValuesBase]] = CPFactorizedVoxelGridValues + + n_components: int = 24 + basis_matrix: bool = True + + # pyre-fixme[14]: `evaluate_local` overrides method defined in `VoxelGridBase` + # inconsistently. + def evaluate_local( + self, points: torch.Tensor, grid_values: CPFactorizedVoxelGridValues + ) -> torch.Tensor: + def factor(axis): + i = {"x": 0, "y": 1, "z": 2}[axis] + index = points[..., i, None] + vector = getattr(grid_values, "vector_components_" + axis) + return interpolate_line( + index, + vector, + align_corners=self.align_corners, + padding_mode=self.padding, + mode=self.mode, + ) + + # (n_grids, n_points_total, n_features) from (n_grids, ..., n_features) + recorded_shape = points.shape + points = points.view(points.shape[0], -1, points.shape[-1]) + + # collect points from all the vectors and multipy them out + mult = factor("x") * factor("y") * factor("z") + + # reduce the result from + # (n_grids, n_points_total, n_components) to (n_grids, n_points_total, n_features) + if grid_values.basis_matrix is not None: + # (n_grids, n_points_total, n_features) = + # (n_grids, n_points_total, total_n_components) @ + # (n_grids, total_n_components, n_features) + result = torch.bmm(mult, grid_values.basis_matrix) + else: + # (n_grids, n_points_total, 1) from (n_grids, n_points_total, n_features) + result = mult.sum(axis=-1, keepdim=True) + # (n_grids, ..., n_features) + return result.view(*recorded_shape[:-1], -1) + + def get_shapes(self, epoch: int) -> Dict[str, Tuple[int, int]]: + if self.basis_matrix is False and self.n_features != 1: + raise ValueError("Cannot set basis_matrix=False and n_features to != 1") + + width, height, depth = self.get_resolution(epoch=epoch) + shape_dict = { + "vector_components_x": (self.n_components, width), + "vector_components_y": (self.n_components, height), + "vector_components_z": (self.n_components, depth), + } + if self.basis_matrix: + shape_dict["basis_matrix"] = (self.n_components, self.n_features) + return shape_dict + + # pyre-ignore[14] + def crop_local( + self, + min_point_local: torch.Tensor, + max_point_local: torch.Tensor, + grid_values: CPFactorizedVoxelGridValues, + ) -> CPFactorizedVoxelGridValues: + assert torch.all(min_point_local < max_point_local) + min_point_local = torch.clamp(min_point_local, -1, 1) + max_point_local = torch.clamp(max_point_local, -1, 1) + _, _, width = grid_values.vector_components_x.shape + _, _, height = grid_values.vector_components_y.shape + _, _, depth = grid_values.vector_components_z.shape + resolution = grid_values.vector_components_x.new_tensor([width, height, depth]) + min_point_local01 = (min_point_local + 1) / 2 + max_point_local01 = (max_point_local + 1) / 2 + + if self.align_corners: + minx, miny, minz = torch.floor(min_point_local01 * (resolution - 1)).long() + maxx, maxy, maxz = torch.ceil(max_point_local01 * (resolution - 1)).long() + else: + minx, miny, minz = torch.floor(min_point_local01 * resolution - 0.5).long() + maxx, maxy, maxz = torch.ceil(max_point_local01 * resolution - 0.5).long() + + return CPFactorizedVoxelGridValues( + vector_components_x=grid_values.vector_components_x[:, :, minx : maxx + 1], + vector_components_y=grid_values.vector_components_y[:, :, miny : maxy + 1], + vector_components_z=grid_values.vector_components_z[:, :, minz : maxz + 1], + basis_matrix=grid_values.basis_matrix, + ) + + +@dataclass +class VMFactorizedVoxelGridValues(VoxelGridValuesBase): + vector_components_x: torch.Tensor + vector_components_y: torch.Tensor + vector_components_z: torch.Tensor + matrix_components_xy: torch.Tensor + matrix_components_yz: torch.Tensor + matrix_components_xz: torch.Tensor + basis_matrix: Optional[torch.Tensor] = None + + +@registry.register +class VMFactorizedVoxelGrid(VoxelGridBase): + """ + Implementation of Vector-Matrix Factorization of a tensor from + https://arxiv.org/abs/2203.09517. + + Vector-Matrix Factorization factorizes the 3d grid into three matrices + (xy, xz, yz) and three vectors (x, y, z). For n_components=1, the 3d grid + is a sum of the outer products (call it ⊗) of each matrix with its + complementary vector: + + 3d_grid = xy ⊗ z + xz ⊗ y + yz ⊗ x. + + These tensors are passed in a VMFactorizedVoxelGridValues object (here obj) + as obj.matrix_components_xy, obj.matrix_components_xy, obj.vector_components_y, etc. + + Their shapes are `(n_grids, n_components, r0, r1)` for matrix_components and + (n_grids, n_components, r2)` for vector_componenets. Each of `r0, r1 and r2` coresponds + to one resolution in (width, height and depth). + + Each element of this sum has an extra dimension, which gets matrix-multiplied by an + appropriate "basis matrix" of shape (n_grids, n_components, n_features). This multiplication + brings us to the desired "n_features" dimensionality. If basis_matrix=False the elements + of different components are summed together to create (n_grids, n_components, 1) tensor. + With some notation abuse, ignoring the interpolation operation, simplifying and denoting + n_features as F, n_components as C (which can differ for each dimension) and n_grids as G: + + 3d_grid = concat((xy ⊗ z), (xz ⊗ y).permute(0, 2, 1), + (yz ⊗ x).permute(2, 0, 1)) @ basis_matrix # GWHDC x GCF -> GWHDF + + Members: + n_components: total number of matrix vector pairs, this must be divisible by 3. Set + this if you want to have equal representational power in all 3 directions. You + must specify either n_components or distribution_of_components, you cannot + specify both. + distribution_of_components: if you do not want equal representational power in + all 3 directions specify a tuple of numbers of matrix_vector pairs for each + coordinate of a form (n_xy_planes, n_yz_planes, n_xz_planes). You must specify + either n_components or distribution_of_components, you cannot specify both. + basis_matrix: how to transform components. If matrix_reduction=True result + matrix of shape (n_grids, n_points_total, n_components) is batch matrix multiplied + by the basis_matrix of shape (n_grids, n_components, n_features). If + matrix_reduction=False, the result tensor of (n_grids, n_points_total, n_components) + is summed along the rows to get (n_grids, n_points_total, 1), which is then viewed + to return to starting shape (n_grids, ..., 1). + """ + + # the type of grid_values argument needed to run evaluate_local() + values_type: ClassVar[Type[VoxelGridValuesBase]] = VMFactorizedVoxelGridValues + + n_components: Optional[int] = None + distribution_of_components: Optional[Tuple[int, int, int]] = None + basis_matrix: bool = True + + # pyre-fixme[14]: `evaluate_local` overrides method defined in `VoxelGridBase` + # inconsistently. + def evaluate_local( + self, points: torch.Tensor, grid_values: VMFactorizedVoxelGridValues + ) -> torch.Tensor: + # (n_grids, n_points_total, n_features) from (n_grids, ..., n_features) + recorded_shape = points.shape + points = points.view(points.shape[0], -1, points.shape[-1]) + + # collect points from matrices and vectors and multiply them + a = interpolate_plane( + points[..., :2], + grid_values.matrix_components_xy, + align_corners=self.align_corners, + padding_mode=self.padding, + mode=self.mode, + ) * interpolate_line( + points[..., 2:], + grid_values.vector_components_z, + align_corners=self.align_corners, + padding_mode=self.padding, + mode=self.mode, + ) + b = interpolate_plane( + points[..., [0, 2]], + grid_values.matrix_components_xz, + align_corners=self.align_corners, + padding_mode=self.padding, + mode=self.mode, + ) * interpolate_line( + points[..., 1:2], + grid_values.vector_components_y, + align_corners=self.align_corners, + padding_mode=self.padding, + mode=self.mode, + ) + c = interpolate_plane( + points[..., 1:], + grid_values.matrix_components_yz, + align_corners=self.align_corners, + padding_mode=self.padding, + mode=self.mode, + ) * interpolate_line( + points[..., :1], + grid_values.vector_components_x, + align_corners=self.align_corners, + padding_mode=self.padding, + mode=self.mode, + ) + # pyre-ignore[28] + feats = torch.cat((a, b, c), axis=-1) + + # reduce the result from + # (n_grids, n_points, n_components) to (n_grids, n_points, n_features) + if grid_values.basis_matrix is not None: + # (n_grids, n_points, n_features) = + # (n_grids, n_points, total_n_components) x + # (n_grids, total_n_components, n_features) + result = torch.bmm(feats, grid_values.basis_matrix) + else: + # pyre-ignore[28] + # (n_grids, n_points, 1) from (n_grids, n_points, n_features) + result = feats.sum(axis=-1, keepdim=True) + # (n_grids, ..., n_features) + return result.view(*recorded_shape[:-1], -1) + + def get_shapes(self, epoch: int) -> Dict[str, Tuple]: + if self.basis_matrix is False and self.n_features != 1: + raise ValueError("Cannot set basis_matrix=False and n_features to != 1") + if self.distribution_of_components is None and self.n_components is None: + raise ValueError( + "You need to provide n_components or distribution_of_components" + ) + if ( + self.distribution_of_components is not None + and self.n_components is not None + ): + raise ValueError( + "You cannot define n_components and distribution_of_components" + ) + # pyre-ignore[58] + if self.distribution_of_components is None and self.n_components % 3 != 0: + raise ValueError("n_components must be divisible by 3") + if self.distribution_of_components is None: + calculated_distribution_of_components = [ + # pyre-fixme[58]: `//` is not supported for operand types + # `Optional[int]` and `int`. + self.n_components // 3 + for _ in range(3) + ] + else: + calculated_distribution_of_components = self.distribution_of_components + + width, height, depth = self.get_resolution(epoch=epoch) + shape_dict = { + "vector_components_x": ( + calculated_distribution_of_components[1], + width, + ), + "vector_components_y": ( + calculated_distribution_of_components[2], + height, + ), + "vector_components_z": ( + calculated_distribution_of_components[0], + depth, + ), + "matrix_components_xy": ( + calculated_distribution_of_components[0], + width, + height, + ), + "matrix_components_yz": ( + calculated_distribution_of_components[1], + height, + depth, + ), + "matrix_components_xz": ( + calculated_distribution_of_components[2], + width, + depth, + ), + } + if self.basis_matrix: + shape_dict["basis_matrix"] = ( + sum(calculated_distribution_of_components), + self.n_features, + ) + + return shape_dict + + # pyre-ignore[14] + def crop_local( + self, + min_point_local: torch.Tensor, + max_point_local: torch.Tensor, + grid_values: VMFactorizedVoxelGridValues, + ) -> VMFactorizedVoxelGridValues: + assert torch.all(min_point_local < max_point_local) + min_point_local = torch.clamp(min_point_local, -1, 1) + max_point_local = torch.clamp(max_point_local, -1, 1) + _, _, width = grid_values.vector_components_x.shape + _, _, height = grid_values.vector_components_y.shape + _, _, depth = grid_values.vector_components_z.shape + resolution = grid_values.vector_components_x.new_tensor([width, height, depth]) + min_point_local01 = (min_point_local + 1) / 2 + max_point_local01 = (max_point_local + 1) / 2 + + if self.align_corners: + minx, miny, minz = torch.floor(min_point_local01 * (resolution - 1)).long() + maxx, maxy, maxz = torch.ceil(max_point_local01 * (resolution - 1)).long() + else: + minx, miny, minz = torch.floor(min_point_local01 * resolution - 0.5).long() + maxx, maxy, maxz = torch.ceil(max_point_local01 * resolution - 0.5).long() + + return VMFactorizedVoxelGridValues( + vector_components_x=grid_values.vector_components_x[:, :, minx : maxx + 1], + vector_components_y=grid_values.vector_components_y[:, :, miny : maxy + 1], + vector_components_z=grid_values.vector_components_z[:, :, minz : maxz + 1], + matrix_components_xy=grid_values.matrix_components_xy[ + :, :, minx : maxx + 1, miny : maxy + 1 + ], + matrix_components_yz=grid_values.matrix_components_yz[ + :, :, miny : maxy + 1, minz : maxz + 1 + ], + matrix_components_xz=grid_values.matrix_components_xz[ + :, :, minx : maxx + 1, minz : maxz + 1 + ], + basis_matrix=grid_values.basis_matrix, + ) + + +# pyre-fixme[13]: Attribute `voxel_grid` is never initialized. +class VoxelGridModule(Configurable, torch.nn.Module): + """ + A wrapper torch.nn.Module for the VoxelGrid classes, which + contains parameters that are needed to train the VoxelGrid classes. + Can contain the parameters for the voxel grid as pytorch parameters + or as registered buffers. + + Members: + voxel_grid_class_type: The name of the class to use for voxel_grid, + which must be available in the registry. Default FullResolutionVoxelGrid. + voxel_grid: An instance of `VoxelGridBase`. This is the object which + this class wraps. + extents: 3-tuple of a form (width, height, depth), denotes the size of the grid + in world units. + translation: 3-tuple of float. The center of the volume in world units as (x, y, z). + init_std: Parameters are initialized using the gaussian distribution + with mean=init_mean and std=init_std. Default 0.1 + init_mean: Parameters are initialized using the gaussian distribution + with mean=init_mean and std=init_std. Default 0. + hold_voxel_grid_as_parameters: if True components of the underlying voxel grids + will be saved as parameters and therefore be trainable. Default True. + param_groups: dictionary where keys are names of individual parameters + or module members and values are the parameter group where the + parameter/member will be sorted to. "self" key is used to denote the + parameter group at the module level. Possible keys, including the "self" key + do not have to be defined. By default all parameters are put into "default" + parameter group and have the learning rate defined in the optimizer, + it can be overridden at the: + - module level with “self” key, all the parameters and child + module's parameters will be put to that parameter group + - member level, which is the same as if the `param_groups` in that + member has key=“self” and value equal to that parameter group. + This is useful if members do not have `param_groups`, for + example torch.nn.Linear. + - parameter level, parameter with the same name as the key + will be put to that parameter group. + """ + + voxel_grid_class_type: str = "FullResolutionVoxelGrid" + voxel_grid: VoxelGridBase + + extents: Tuple[float, float, float] = (2.0, 2.0, 2.0) + translation: Tuple[float, float, float] = (0.0, 0.0, 0.0) + + init_std: float = 0.1 + init_mean: float = 0 + + hold_voxel_grid_as_parameters: bool = True + param_groups: Dict[str, str] = field(default_factory=lambda: {}) + + def __post_init__(self): + run_auto_creation(self) + n_grids = 1 # Voxel grid objects are batched. We need only a single grid. + shapes = self.voxel_grid.get_shapes(epoch=0) + params = { + name: torch.normal( + mean=torch.zeros((n_grids, *shape)) + self.init_mean, + std=self.init_std, + ) + for name, shape in shapes.items() + } + + self.set_voxel_grid_parameters(self.voxel_grid.values_type(**params)) + self._register_load_state_dict_pre_hook(self._create_parameters_with_new_size) + + def forward(self, points: torch.Tensor) -> torch.Tensor: + """ + Evaluates points in the world coordinate frame on the voxel_grid. + + Args: + points (torch.Tensor): tensor of points that you want to query + of a form (..., 3) + Returns: + torch.Tensor of shape (..., n_features) + """ + locator = self._get_volume_locator() + grid_values = self.voxel_grid.values_type(**self.params) + # voxel grids operate with extra n_grids dimension, which we fix to one + return self.voxel_grid.evaluate_world(points[None], grid_values, locator)[0] + + def set_voxel_grid_parameters(self, params: VoxelGridValuesBase) -> None: + """ + Sets the parameters of the underlying voxel grid. + + Args: + params: parameters of type `self.voxel_grid.values_type` which will + replace current parameters + """ + if self.hold_voxel_grid_as_parameters: + self.params = torch.nn.ParameterDict( + { + k: torch.nn.Parameter(val) + for k, val in vars(params).items() + if val is not None + } + ) + else: + # Torch Module to hold parameters since they can only be registered + # at object level. + self.params = _RegistratedBufferDict(vars(params)) + + @staticmethod + def get_output_dim(args: DictConfig) -> int: + """ + Utility to help predict the shape of the output of `forward`. + + Args: + args: DictConfig which would be used to initialize the object + Returns: + int: the length of the last dimension of the output tensor + """ + grid = registry.get(VoxelGridBase, args["voxel_grid_class_type"]) + return grid.get_output_dim( + args["voxel_grid_" + args["voxel_grid_class_type"] + "_args"] + ) + + def subscribe_to_epochs(self) -> Tuple[Tuple[int, ...], Callable[[int], bool]]: + """ + Method which expresses interest in subscribing to optimization epoch updates. + + Returns: + tuple of epochs on which to call a callable and callable to be called on + particular epoch. The callable returns True if parameter change has + happened else False and it must be supplied with one argument, epoch. + """ + return self.voxel_grid.get_resolution_change_epochs(), self._apply_epochs + + def _apply_epochs(self, epoch: int) -> bool: + """ + Asks voxel_grid to change the resolution. + This method is returned with subscribe_to_epochs and is the method that collects + updates on training epochs, it is run on the training epochs that are requested. + + Args: + epoch: current training epoch used for voxel grids to know to which + resolution to change + Returns: + True if parameter change has happened else False. + """ + grid_values = self.voxel_grid.values_type(**self.params) + grid_values, change = self.voxel_grid.change_resolution( + grid_values, epoch=epoch + ) + if change: + self.set_voxel_grid_parameters(grid_values) + return change and self.hold_voxel_grid_as_parameters + + def _create_parameters_with_new_size( + self, + state_dict: dict, + prefix: str, + local_metadata: dict, + strict: bool, + missing_keys: List[str], + unexpected_keys: List[str], + error_msgs: List[str], + ) -> None: + ''' + Automatically ran before loading the parameters with `load_state_dict()`. + Creates new parameters with the sizes of the ones in the loaded state dict. + This is necessary because the parameters are changing throughout training and + at the time of construction `VoxelGridModule` does not know the size of + parameters which will be loaded. + + Args: + state_dict (dict): a dict containing parameters and + persistent buffers. + prefix (str): the prefix for parameters and buffers used in this + module + local_metadata (dict): a dict containing the metadata for this module. + See + strict (bool): whether to strictly enforce that the keys in + :attr:`state_dict` with :attr:`prefix` match the names of + parameters and buffers in this module + missing_keys (list of str): if ``strict=True``, add missing keys to + this list + unexpected_keys (list of str): if ``strict=True``, add unexpected + keys to this list + error_msgs (list of str): error messages should be added to this + list, and will be reported together in + :meth:`~torch.nn.Module.load_state_dict` + Returns: + nothing + """ + ''' + new_params = {} + for name in self.params: + key = prefix + "params." + name + if key in state_dict: + new_params[name] = torch.zeros_like(state_dict[key]) + self.set_voxel_grid_parameters(self.voxel_grid.values_type(**new_params)) + + def get_device(self) -> torch.device: + """ + Returns torch.device on which module parameters are located + """ + return next(val for val in self.params.values() if val is not None).device + + def crop_self(self, min_point: torch.Tensor, max_point: torch.Tensor) -> None: + """ + Crops self to only represent points between min_point and max_point (inclusive). + + Args: + min_point: torch.Tensor of shape (3,). Has x, y and z coordinates + smaller or equal to all other occupied points. + max_point: torch.Tensor of shape (3,). Has x, y and z coordinates + bigger or equal to all other occupied points. + Returns: + nothing + """ + locator = self._get_volume_locator() + # torch.nn.modules.module.Module]` is not a function. + old_grid_values = self.voxel_grid.values_type(**self.params) + new_grid_values = self.voxel_grid.crop_world( + min_point, max_point, old_grid_values, locator + ) + grid_values, _ = self.voxel_grid.change_resolution( + new_grid_values, grid_values_with_wanted_resolution=old_grid_values + ) + self.params = torch.nn.ParameterDict( + { + k: torch.nn.Parameter(val) + for k, val in vars(grid_values).items() + if val is not None + } + ) + # New center of voxel grid is the middle point between max and min points. + self.translation = tuple((max_point + min_point) / 2) + # new extents of voxel grid are distances between min and max points + self.extents = tuple(max_point - min_point) + + def _get_volume_locator(self) -> VolumeLocator: + """ + Returns VolumeLocator calculated from `extents` and `translation` members. + """ + return VolumeLocator( + batch_size=1, + # The resolution of the voxel grid does not need to be known + # to the locator object. It is easiest to fix the resolution of the locator. + # In particular we fix it to (2,2,2) so that there is exactly one voxel of the + # desired size. The locator object uses (z, y, x) convention for the grid_size, + # and this module uses (x, y, z) convention so the order has to be reversed + # (irrelevant in this case since they are all equal). + # It is (2, 2, 2) because the VolumeLocator object behaves like + # align_corners=True, which means that the points are in the corners of + # the volume. So in the grid of (2, 2, 2) there is only one voxel. + grid_sizes=(2, 2, 2), + # The locator object uses (x, y, z) convention for the + # voxel size and translation. + voxel_size=tuple(self.extents), + # volume_translation is defined in `VolumeLocator` as a vector from the origin + # of local coordinate frame to origin of world coordinate frame, that is: + # x_world = x_local * extents/2 - translation. + # To get the reverse we need to negate it. + volume_translation=tuple(-t for t in self.translation), + device=self.get_device(), + ) + + def get_grid_points(self, epoch: int) -> torch.Tensor: + """ + Returns a grid of points that represent centers of voxels of the + underlying voxel grid in world coordinates at specific epoch. + + Args: + epoch: underlying voxel grids change resolution depending on the + epoch, this argument is used to determine the resolution + of the voxel grid at that epoch. + Returns: + tensor of shape [xresolution, yresolution, zresolution, 3] where + xresolution, yresolution, zresolution are resolutions of the + underlying voxel grid + """ + xresolution, yresolution, zresolution = self.voxel_grid.get_resolution(epoch) + width, height, depth = self.extents + if not self.voxel_grid.get_align_corners(): + width = ( + width * (xresolution - 1) / xresolution if xresolution > 1 else width + ) + height = ( + height * (xresolution - 1) / xresolution if xresolution > 1 else height + ) + depth = ( + depth * (xresolution - 1) / xresolution if xresolution > 1 else depth + ) + xs = torch.linspace( + -width / 2, width / 2, xresolution, device=self.get_device() + ) + ys = torch.linspace( + -height / 2, height / 2, yresolution, device=self.get_device() + ) + zs = torch.linspace( + -depth / 2, depth / 2, zresolution, device=self.get_device() + ) + xmesh, ymesh, zmesh = torch.meshgrid(xs, ys, zs, indexing="ij") + return torch.stack((xmesh, ymesh, zmesh), dim=3) + + +class _RegistratedBufferDict(torch.nn.Module, Mapping): + """ + Mapping class and a torch.nn.Module that registeres its values + with `self.register_buffer`. Can be indexed like a regular Python + dictionary, but torch.Tensors it contains are properly registered, and will be visible + by all Module methods. Supports only `torch.Tensor` as value and str as key. + """ + + def __init__(self, init_dict: Optional[Dict[str, torch.Tensor]] = None) -> None: + """ + Args: + init_dict: dictionary which will be used to populate the object + """ + super().__init__() + self._keys = set() + if init_dict is not None: + for k, v in init_dict.items(): + self[k] = v + + def __iter__(self) -> Iterator[Dict[str, torch.Tensor]]: + return iter({k: self[k] for k in self._keys}) + + def __len__(self) -> int: + return len(self._keys) + + def __getitem__(self, key: str) -> torch.Tensor: + return getattr(self, key) + + def __setitem__(self, key, value) -> None: + self._keys.add(key) + self.register_buffer(key, value) + + def __hash__(self) -> int: + return hash(repr(self)) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/voxel_grid_implicit_function.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/voxel_grid_implicit_function.py new file mode 100644 index 0000000000000000000000000000000000000000..6b9b27b70cdef3033e8ea6e7e4b32696889af5dd --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/implicit_function/voxel_grid_implicit_function.py @@ -0,0 +1,618 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import logging +import math +import warnings +from dataclasses import fields +from typing import Callable, Dict, Optional, Tuple + +import torch + +from omegaconf import DictConfig + +from pytorch3d.implicitron.models.implicit_function.base import ImplicitFunctionBase +from pytorch3d.implicitron.models.implicit_function.decoding_functions import ( + DecoderFunctionBase, +) +from pytorch3d.implicitron.models.implicit_function.voxel_grid import VoxelGridModule +from pytorch3d.implicitron.models.renderer.base import ImplicitronRayBundle +from pytorch3d.implicitron.tools.config import ( + enable_get_default_args, + get_default_args_field, + registry, + run_auto_creation, +) +from pytorch3d.renderer import ray_bundle_to_ray_points +from pytorch3d.renderer.cameras import CamerasBase +from pytorch3d.renderer.implicit import HarmonicEmbedding + +logger = logging.getLogger(__name__) + + +enable_get_default_args(HarmonicEmbedding) + + +@registry.register +# pyre-ignore[13] +class VoxelGridImplicitFunction(ImplicitFunctionBase, torch.nn.Module): + """ + This implicit function consists of two streams, one for the density calculation and one + for the color calculation. Each of these streams has three main parts: + 1) Voxel grids: + They take the (x, y, z) position and return the embedding of that point. + These components are replaceable, you can make your own or choose one of + several options. + 2) Harmonic embeddings: + Convert each feature into series of 'harmonic features', feature is passed through + sine and cosine functions. Input is of shape [minibatch, ..., D] output + [minibatch, ..., (n_harmonic_functions * 2 + int(append_input)) * D]. Appends + input by default. If you want it to behave like identity, put n_harmonic_functions=0 + and append_input=True. + 3) Decoding functions: + The decoder is an instance of the DecoderFunctionBase and converts the embedding + of a spatial location to density/color. Examples are Identity which returns its + input and the MLP which uses fully connected nerual network to transform the input. + These components are replaceable, you can make your own or choose from + several options. + + Calculating density is done in three steps: + 1) Evaluating the voxel grid on points + 2) Embedding the outputs with harmonic embedding + 3) Passing through the Density decoder + + To calculate the color we need the embedding and the viewing direction, it has five steps: + 1) Transforming the viewing direction with camera + 2) Evaluating the voxel grid on points + 3) Embedding the outputs with harmonic embedding + 4) Embedding the normalized direction with harmonic embedding + 5) Passing everything through the Color decoder + + If using the Implicitron configuration system the input_dim to the decoding functions will + be set to the output_dim of the Harmonic embeddings. + + A speed up comes from using the scaffold, a low resolution voxel grid. + The scaffold is referenced as "binary occupancy grid mask" in TensoRF paper and "AlphaMask" + in official TensoRF implementation. + The scaffold is used in: + 1) filtering points in empty space + - controlled by `scaffold_filter_points` boolean. If set to True, points for which + scaffold predicts that are in empty space will return 0 density and + (0, 0, 0) color. + 2) calculating the bounding box of an object and cropping the voxel grids + - controlled by `volume_cropping_epochs`. + - at those epochs the implicit function will find the bounding box of an object + inside it and crop density and color grids. Cropping of the voxel grids means + preserving only voxel values that are inside the bounding box and changing the + resolution to match the original, while preserving the new cropped location in + world coordinates. + + The scaffold has to exist before attempting filtering and cropping, and is created on + `scaffold_calculating_epochs`. Each voxel in the scaffold is labeled as having density 1 if + the point in the center of it evaluates to greater than `scaffold_empty_space_threshold`. + 3D max pooling is performed on the densities of the points in 3D. + Scaffold features are off by default. + + Members: + voxel_grid_density (VoxelGridBase): voxel grid to use for density estimation + voxel_grid_color (VoxelGridBase): voxel grid to use for color estimation + + harmonic_embedder_xyz_density (HarmonicEmbedder): Function to transform the outputs of + the voxel_grid_density + harmonic_embedder_xyz_color (HarmonicEmbedder): Function to transform the outputs of + the voxel_grid_color for density + harmonic_embedder_dir_color (HarmonicEmbedder): Function to transform the outputs of + the voxel_grid_color for color + + decoder_density (DecoderFunctionBase): decoder function to use for density estimation + color_density (DecoderFunctionBase): decoder function to use for color estimation + + use_multiple_streams (bool): if you want the density and color calculations to run on + different cuda streams set this to True. Default True. + xyz_ray_dir_in_camera_coords (bool): This is true if the directions are given in + camera coordinates. Default False. + + voxel_grid_scaffold (VoxelGridModule): which holds the scaffold. Extents and + translation of it are set to those of voxel_grid_density. + scaffold_calculating_epochs (Tuple[int, ...]): at which epochs to recalculate the + scaffold. (The scaffold will be created automatically at the beginning of + the calculation.) + scaffold_resolution (Tuple[int, int, int]): (width, height, depth) of the underlying + voxel grid which stores scaffold + scaffold_empty_space_threshold (float): if `self._get_density` evaluates to less than + this it will be considered as empty space and the scaffold at that point would + evaluate as empty space. + scaffold_occupancy_chunk_size (str or int): Number of xy scaffold planes to calculate + at the same time. To calculate the scaffold we need to query `_get_density()` at + every voxel, this calculation can be split into scaffold depth number of xy plane + calculations if you want the lowest memory usage, one calculation to calculate the + whole scaffold, but with higher memory footprint or any other number of planes. + Setting to a non-positive number calculates all planes at the same time. + Defaults to '-1' (=calculating all planes). + scaffold_max_pool_kernel_size (int): Size of the pooling region to use when + calculating the scaffold. Defaults to 3. + scaffold_filter_points (bool): If set to True the points will be filtered using + `self.voxel_grid_scaffold`. Filtered points will be predicted as having 0 density + and (0, 0, 0) color. The points which were not evaluated as empty space will be + passed through the steps outlined above. + volume_cropping_epochs: on which epochs to crop the voxel grids to fit the object's + bounding box. Scaffold has to be calculated before cropping. + """ + + # ---- voxel grid for density + voxel_grid_density: VoxelGridModule + + # ---- voxel grid for color + voxel_grid_color: VoxelGridModule + + # ---- harmonic embeddings density + harmonic_embedder_xyz_density_args: DictConfig = get_default_args_field( + HarmonicEmbedding + ) + harmonic_embedder_xyz_color_args: DictConfig = get_default_args_field( + HarmonicEmbedding + ) + harmonic_embedder_dir_color_args: DictConfig = get_default_args_field( + HarmonicEmbedding + ) + + # ---- decoder function for density + decoder_density_class_type: str = "MLPDecoder" + decoder_density: DecoderFunctionBase + + # ---- decoder function for color + decoder_color_class_type: str = "MLPDecoder" + decoder_color: DecoderFunctionBase + + # ---- cuda streams + use_multiple_streams: bool = True + + # ---- camera + xyz_ray_dir_in_camera_coords: bool = False + + # --- scaffold + # voxel_grid_scaffold: VoxelGridModule + scaffold_calculating_epochs: Tuple[int, ...] = () + scaffold_resolution: Tuple[int, int, int] = (128, 128, 128) + scaffold_empty_space_threshold: float = 0.001 + scaffold_occupancy_chunk_size: int = -1 + scaffold_max_pool_kernel_size: int = 3 + scaffold_filter_points: bool = True + + # --- cropping + volume_cropping_epochs: Tuple[int, ...] = () + + def __post_init__(self) -> None: + run_auto_creation(self) + self.voxel_grid_scaffold = self._create_voxel_grid_scaffold() + self.harmonic_embedder_xyz_density = HarmonicEmbedding( + **self.harmonic_embedder_xyz_density_args + ) + self.harmonic_embedder_xyz_color = HarmonicEmbedding( + **self.harmonic_embedder_xyz_color_args + ) + self.harmonic_embedder_dir_color = HarmonicEmbedding( + **self.harmonic_embedder_dir_color_args + ) + self._scaffold_ready = False + + def forward( + self, + ray_bundle: ImplicitronRayBundle, + fun_viewpool=None, + camera: Optional[CamerasBase] = None, + global_code=None, + **kwargs, + ) -> Tuple[torch.Tensor, torch.Tensor, Dict]: + """ + The forward function accepts the parametrizations of 3D points sampled along + projection rays. The forward pass is responsible for attaching a 3D vector + and a 1D scalar representing the point's RGB color and opacity respectively. + + Args: + ray_bundle: An ImplicitronRayBundle object containing the following variables: + origins: A tensor of shape `(minibatch, ..., 3)` denoting the + origins of the sampling rays in world coords. + directions: A tensor of shape `(minibatch, ..., 3)` + containing the direction vectors of sampling rays in world coords. + lengths: A tensor of shape `(minibatch, ..., num_points_per_ray)` + containing the lengths at which the rays are sampled. + fun_viewpool: an optional callback with the signature + fun_fiewpool(points) -> pooled_features + where points is a [N_TGT x N x 3] tensor of world coords, + and pooled_features is a [N_TGT x ... x N_SRC x latent_dim] tensor + of the features pooled from the context images. + camera: A camera model which will be used to transform the viewing + directions + + Returns: + rays_densities: A tensor of shape `(minibatch, ..., num_points_per_ray, 1)` + denoting the opacitiy of each ray point. + rays_colors: A tensor of shape `(minibatch, ..., num_points_per_ray, 3)` + denoting the color of each ray point. + """ + # ########## convert the ray parametrizations to world coordinates ########## # + # points.shape = [minibatch x n_rays_width x n_rays_height x pts_per_ray x 3] + # pyre-ignore[6] + points = ray_bundle_to_ray_points(ray_bundle) + directions = ray_bundle.directions.reshape(-1, 3) + input_shape = points.shape + num_points_per_ray = input_shape[-2] + points = points.view(-1, 3) + non_empty_points = None + + # ########## filter the points using the scaffold ########## # + if self._scaffold_ready and self.scaffold_filter_points: + with torch.no_grad(): + non_empty_points = self.voxel_grid_scaffold(points)[..., 0] > 0 + points = points[non_empty_points] + if len(points) == 0: + warnings.warn( + "The scaffold has filtered all the points." + "The voxel grids and decoding functions will not be run." + ) + return ( + points.new_zeros((*input_shape[:-1], 1)), + points.new_zeros((*input_shape[:-1], 3)), + {}, + ) + + # ########## calculate color and density ########## # + rays_densities, rays_colors = self._calculate_density_and_color( + points, directions, camera, non_empty_points, num_points_per_ray + ) + + if not (self._scaffold_ready and self.scaffold_filter_points): + return ( + rays_densities.view((*input_shape[:-1], rays_densities.shape[-1])), + rays_colors.view((*input_shape[:-1], rays_colors.shape[-1])), + {}, + ) + + # ########## merge scaffold calculated points ########## # + # Create a zeroed tensor corresponding to a point with density=0 and fill it + # with calculated density for points which are not in empty space. Do the + # same for color + rays_densities_combined = rays_densities.new_zeros( + (math.prod(input_shape[:-1]), rays_densities.shape[-1]) + ) + rays_colors_combined = rays_colors.new_zeros( + (math.prod(input_shape[:-1]), rays_colors.shape[-1]) + ) + assert non_empty_points is not None + rays_densities_combined[non_empty_points] = rays_densities + rays_colors_combined[non_empty_points] = rays_colors + + return ( + rays_densities_combined.view((*input_shape[:-1], rays_densities.shape[-1])), + rays_colors_combined.view((*input_shape[:-1], rays_colors.shape[-1])), + {}, + ) + + def _calculate_density_and_color( + self, + points: torch.Tensor, + directions: torch.Tensor, + camera: Optional[CamerasBase], + non_empty_points: Optional[torch.Tensor], + num_points_per_ray: int, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Calculates density and color at `points`. + If enabled use cuda streams. + + Args: + points: points at which to calculate density and color. + Tensor of shape [n_points, 3]. + directions: from which directions are the points viewed. + One per ray. Tensor of shape [n_rays, 3]. + camera: A camera model which will be used to transform the viewing + directions + non_empty_points: indices of points which weren't filtered out; + used for expanding directions + num_points_per_ray: number of points per ray, needed to expand directions. + Returns: + Tuple of color (tensor of shape [..., 3]) and density + (tensor of shape [..., 1]) + """ + if self.use_multiple_streams and points.is_cuda: + current_stream = torch.cuda.current_stream(points.device) + other_stream = torch.cuda.Stream(points.device) + other_stream.wait_stream(current_stream) + + with torch.cuda.stream(other_stream): + # rays_densities.shape = + # [minibatch x n_rays_width x n_rays_height x pts_per_ray x density_dim] + rays_densities = self._get_density(points) + + # rays_colors.shape = + # [minibatch x n_rays_width x n_rays_height x pts_per_ray x color_dim] + rays_colors = self._get_color( + points, camera, directions, non_empty_points, num_points_per_ray + ) + + current_stream.wait_stream(other_stream) + else: + # Same calculation as above, just serial. + rays_densities = self._get_density(points) + rays_colors = self._get_color( + points, camera, directions, non_empty_points, num_points_per_ray + ) + return rays_densities, rays_colors + + def _get_density(self, points: torch.Tensor) -> torch.Tensor: + """ + Calculates density at points: + 1) Evaluates the voxel grid on points + 2) Embeds the outputs with harmonic embedding + 3) Passes everything through the Density decoder + + Args: + points: tensor of shape [..., 3] + where the last dimension is the points in the (x, y, z) + Returns: + calculated densities of shape [..., density_dim], `density_dim` is the + feature dimensionality which `decoder_density` returns + """ + embeds_density = self.voxel_grid_density(points) + harmonic_embedding_density = self.harmonic_embedder_xyz_density(embeds_density) + # shape = [..., density_dim] + return self.decoder_density(harmonic_embedding_density) + + def _get_color( + self, + points: torch.Tensor, + camera: Optional[CamerasBase], + directions: torch.Tensor, + non_empty_points: Optional[torch.Tensor], + num_points_per_ray: int, + ) -> torch.Tensor: + """ + Calculates color at points using the viewing direction: + 1) Transforms the viewing direction with camera + 2) Evaluates the voxel grid on points + 3) Embeds the outputs with harmonic embedding + 4) Embeds the normalized direction with harmonic embedding + 5) Passes everything through the Color decoder + Args: + points: tensor of shape (..., 3) + where the last dimension is the points in the (x, y, z) + camera: A camera model which will be used to transform the viewing + directions + directions: A tensor of shape `(..., 3)` + containing the direction vectors of sampling rays in world coords. + non_empty_points: indices of points which weren't filtered out; + used for expanding directions + num_points_per_ray: number of points per ray, needed to expand directions. + """ + # ########## transform direction ########## # + if self.xyz_ray_dir_in_camera_coords: + if camera is None: + raise ValueError("Camera must be given if xyz_ray_dir_in_camera_coords") + directions = directions @ camera.R + + # ########## get voxel grid output ########## # + # embeds_color.shape = [..., pts_per_ray, n_features] + embeds_color = self.voxel_grid_color(points) + + # ########## embed with the harmonic function ########## # + # Obtain the harmonic embedding of the voxel grid output. + harmonic_embedding_color = self.harmonic_embedder_xyz_color(embeds_color) + + # Normalize the ray_directions to unit l2 norm. + rays_directions_normed = torch.nn.functional.normalize(directions, dim=-1) + # Obtain the harmonic embedding of the normalized ray directions. + harmonic_embedding_dir = self.harmonic_embedder_dir_color( + rays_directions_normed + ) + + harmonic_embedding_dir = torch.repeat_interleave( + harmonic_embedding_dir, num_points_per_ray, dim=0 + ) + if non_empty_points is not None: + harmonic_embedding_dir = harmonic_embedding_dir[non_empty_points] + + # total color embedding is concatenation of the harmonic embedding of voxel grid + # output and harmonic embedding of the normalized direction + total_color_embedding = torch.cat( + (harmonic_embedding_color, harmonic_embedding_dir), dim=-1 + ) + + # ########## evaluate color with the decoding function ########## # + # rays_colors.shape = [..., pts_per_ray, 3] in [0-1] + return self.decoder_color(total_color_embedding) + + @staticmethod + def allows_multiple_passes() -> bool: + """ + Returns True as this implicit function allows + multiple passes. Overridden from ImplicitFunctionBase. + """ + return True + + def subscribe_to_epochs(self) -> Tuple[Tuple[int, ...], Callable[[int], bool]]: + """ + Method which expresses interest in subscribing to optimization epoch updates. + This implicit function subscribes to epochs to calculate the scaffold and to + crop voxel grids, so this method combines wanted epochs and wraps their callbacks. + + Returns: + list of epochs on which to call a callable and callable to be called on + particular epoch. The callable returns True if parameter change has + happened else False and it must be supplied with one argument, epoch. + """ + + def callback(epoch) -> bool: + change = False + if epoch in self.scaffold_calculating_epochs: + change = self._get_scaffold(epoch) + if epoch in self.volume_cropping_epochs: + change = self._crop(epoch) or change + return change + + # remove duplicates + call_epochs = list( + set(self.scaffold_calculating_epochs) | set(self.volume_cropping_epochs) + ) + return call_epochs, callback + + def _crop(self, epoch: int) -> bool: + """ + Finds the bounding box of an object represented in the scaffold and crops + density and color voxel grids to match that bounding box. If density of the + scaffold is 0 everywhere (there is no object in it) no change will + happen. + + Args: + epoch: ignored + Returns: + True (indicating that parameter change has happened) if there is + an object inside, else False. + """ + # find bounding box + points = self.voxel_grid_scaffold.get_grid_points(epoch=epoch) + assert self._scaffold_ready, "Scaffold has to be calculated before cropping." + occupancy = self.voxel_grid_scaffold(points)[..., 0] > 0 + non_zero_idxs = torch.nonzero(occupancy) + if len(non_zero_idxs) == 0: + return False + min_indices = tuple(torch.min(non_zero_idxs, dim=0)[0]) + max_indices = tuple(torch.max(non_zero_idxs, dim=0)[0]) + min_point, max_point = points[min_indices], points[max_indices] + + logger.info( + f"Cropping at epoch {epoch} to bounding box " + f"[{min_point.tolist()}, {max_point.tolist()}]." + ) + + # crop the voxel grids + self.voxel_grid_density.crop_self(min_point, max_point) + self.voxel_grid_color.crop_self(min_point, max_point) + return True + + @torch.no_grad() + def _get_scaffold(self, epoch: int) -> bool: + """ + Creates a low resolution grid which is used to filter points that are in empty + space. + + Args: + epoch: epoch on which it is called, ignored inside method + Returns: + Always False: Modifies `self.voxel_grid_scaffold` member. + """ + + planes = [] + points = self.voxel_grid_scaffold.get_grid_points(epoch=epoch) + + chunk_size = ( + self.scaffold_occupancy_chunk_size + if self.scaffold_occupancy_chunk_size > 0 + else points.shape[-1] + ) + for k in range(0, points.shape[-1], chunk_size): + points_in_planes = points[..., k : k + chunk_size] + planes.append(self._get_density(points_in_planes)[..., 0]) + + density_cube = torch.cat(planes, dim=-1) + density_cube = torch.nn.functional.max_pool3d( + density_cube[None, None], + kernel_size=self.scaffold_max_pool_kernel_size, + padding=self.scaffold_max_pool_kernel_size // 2, + stride=1, + ) + occupancy_cube = density_cube > self.scaffold_empty_space_threshold + self.voxel_grid_scaffold.params["voxel_grid"] = occupancy_cube.float() + self._scaffold_ready = True + + return False + + @classmethod + def decoder_density_tweak_args(cls, type_, args: DictConfig) -> None: + args.pop("input_dim", None) + + def create_decoder_density_impl(self, type_, args: DictConfig) -> None: + """ + Decoding functions come after harmonic embedding and voxel grid. In order to not + calculate the input dimension of the decoder in the config file this function + calculates the required input dimension and sets the input dimension of the + decoding function to this value. + """ + grid_args = self.voxel_grid_density_args + grid_output_dim = VoxelGridModule.get_output_dim(grid_args) + + embedder_args = self.harmonic_embedder_xyz_density_args + input_dim = HarmonicEmbedding.get_output_dim_static( + grid_output_dim, + embedder_args["n_harmonic_functions"], + embedder_args["append_input"], + ) + + cls = registry.get(DecoderFunctionBase, type_) + need_input_dim = any(field.name == "input_dim" for field in fields(cls)) + if need_input_dim: + self.decoder_density = cls(input_dim=input_dim, **args) + else: + self.decoder_density = cls(**args) + + @classmethod + def decoder_color_tweak_args(cls, type_, args: DictConfig) -> None: + args.pop("input_dim", None) + + def create_decoder_color_impl(self, type_, args: DictConfig) -> None: + """ + Decoding functions come after harmonic embedding and voxel grid. In order to not + calculate the input dimension of the decoder in the config file this function + calculates the required input dimension and sets the input dimension of the + decoding function to this value. + """ + grid_args = self.voxel_grid_color_args + grid_output_dim = VoxelGridModule.get_output_dim(grid_args) + + embedder_args = self.harmonic_embedder_xyz_color_args + input_dim0 = HarmonicEmbedding.get_output_dim_static( + grid_output_dim, + embedder_args["n_harmonic_functions"], + embedder_args["append_input"], + ) + + dir_dim = 3 + embedder_args = self.harmonic_embedder_dir_color_args + input_dim1 = HarmonicEmbedding.get_output_dim_static( + dir_dim, + embedder_args["n_harmonic_functions"], + embedder_args["append_input"], + ) + + input_dim = input_dim0 + input_dim1 + + cls = registry.get(DecoderFunctionBase, type_) + need_input_dim = any(field.name == "input_dim" for field in fields(cls)) + if need_input_dim: + self.decoder_color = cls(input_dim=input_dim, **args) + else: + self.decoder_color = cls(**args) + + def _create_voxel_grid_scaffold(self) -> VoxelGridModule: + """ + Creates object to become self.voxel_grid_scaffold: + - makes `self.voxel_grid_scaffold` have same world to local mapping as + `self.voxel_grid_density` + """ + return VoxelGridModule( + extents=self.voxel_grid_density_args["extents"], + translation=self.voxel_grid_density_args["translation"], + voxel_grid_class_type="FullResolutionVoxelGrid", + hold_voxel_grid_as_parameters=False, + voxel_grid_FullResolutionVoxelGrid_args={ + "resolution_changes": {0: self.scaffold_resolution}, + "padding": "zeros", + "align_corners": True, + "mode": "trilinear", + }, + ) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/metrics.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..0e1a3b6220a3e3e3238dcbc9796b3fa1d7e5d230 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/metrics.py @@ -0,0 +1,402 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +import warnings +from typing import Any, Dict, Optional + +import torch +from pytorch3d.implicitron.models.renderer.ray_sampler import ImplicitronRayBundle +from pytorch3d.implicitron.tools import metric_utils as utils +from pytorch3d.implicitron.tools.config import registry, ReplaceableBase +from pytorch3d.ops import padded_to_packed +from pytorch3d.renderer import utils as rend_utils + +from .renderer.base import RendererOutput + + +class RegularizationMetricsBase(ReplaceableBase, torch.nn.Module): + """ + Replaceable abstract base for regularization metrics. + `forward()` method produces regularization metrics and (unlike ViewMetrics) can + depend on the model's parameters. + """ + + def forward( + self, model: Any, keys_prefix: str = "loss_", **kwargs + ) -> Dict[str, Any]: + """ + Calculates various regularization terms useful for supervising differentiable + rendering pipelines. + + Args: + model: A model instance. Useful, for example, to implement + weights-based regularization. + keys_prefix: A common prefix for all keys in the output dictionary + containing all regularization metrics. + + Returns: + A dictionary with the resulting regularization metrics. The items + will have form `{metric_name_i: metric_value_i}` keyed by the + names of the output metrics `metric_name_i` with their corresponding + values `metric_value_i` represented as 0-dimensional float tensors. + """ + raise NotImplementedError + + +class ViewMetricsBase(ReplaceableBase, torch.nn.Module): + """ + Replaceable abstract base for model metrics. + `forward()` method produces losses and other metrics. + """ + + def forward( + self, + raymarched: RendererOutput, + ray_bundle: ImplicitronRayBundle, + image_rgb: Optional[torch.Tensor] = None, + depth_map: Optional[torch.Tensor] = None, + fg_probability: Optional[torch.Tensor] = None, + mask_crop: Optional[torch.Tensor] = None, + keys_prefix: str = "loss_", + **kwargs, + ) -> Dict[str, Any]: + """ + Calculates various metrics and loss functions useful for supervising + differentiable rendering pipelines. Any additional parameters can be passed + in the `raymarched.aux` dictionary. + + Args: + results: A dictionary with the resulting view metrics. The items + will have form `{metric_name_i: metric_value_i}` keyed by the + names of the output metrics `metric_name_i` with their corresponding + values `metric_value_i` represented as 0-dimensional float tensors. + raymarched: Output of the renderer. + ray_bundle: ImplicitronRayBundle object which was used to produce the raymarched + object + image_rgb: A tensor of shape `(B, H, W, 3)` containing ground truth rgb + values. + depth_map: A tensor of shape `(B, Hd, Wd, 1)` containing ground truth depth + values. + fg_probability: A tensor of shape `(B, Hm, Wm, 1)` containing ground truth + foreground masks. + keys_prefix: A common prefix for all keys in the output dictionary + containing all view metrics. + + Returns: + A dictionary with the resulting view metrics. The items + will have form `{metric_name_i: metric_value_i}` keyed by the + names of the output metrics `metric_name_i` with their corresponding + values `metric_value_i` represented as 0-dimensional float tensors. + """ + raise NotImplementedError() + + +@registry.register +class RegularizationMetrics(RegularizationMetricsBase): + def forward( + self, model: Any, keys_prefix: str = "loss_", **kwargs + ) -> Dict[str, Any]: + """ + Calculates the AD penalty, or returns an empty dict if the model's autoencoder + is inactive. + + Args: + model: A model instance. + keys_prefix: A common prefix for all keys in the output dictionary + containing all regularization metrics. + + Returns: + A dictionary with the resulting regularization metrics. The items + will have form `{metric_name_i: metric_value_i}` keyed by the + names of the output metrics `metric_name_i` with their corresponding + values `metric_value_i` represented as 0-dimensional float tensors. + + The calculated metric is: + autoencoder_norm: Autoencoder weight norm regularization term. + """ + metrics = {} + if getattr(model, "sequence_autodecoder", None) is not None: + ad_penalty = model.sequence_autodecoder.calculate_squared_encoding_norm() + if ad_penalty is not None: + metrics["autodecoder_norm"] = ad_penalty + + if keys_prefix is not None: + metrics = {(keys_prefix + k): v for k, v in metrics.items()} + + return metrics + + +@registry.register +class ViewMetrics(ViewMetricsBase): + def forward( + self, + raymarched: RendererOutput, + ray_bundle: ImplicitronRayBundle, + image_rgb: Optional[torch.Tensor] = None, + depth_map: Optional[torch.Tensor] = None, + fg_probability: Optional[torch.Tensor] = None, + mask_crop: Optional[torch.Tensor] = None, + keys_prefix: str = "loss_", + **kwargs, + ) -> Dict[str, Any]: + """ + Calculates various differentiable metrics useful for supervising + differentiable rendering pipelines. + + Args: + results: A dict to store the results in. + raymarched.features: Predicted rgb or feature values. + raymarched.depths: A tensor of shape `(B, ..., 1)` containing + predicted depth values. + raymarched.masks: A tensor of shape `(B, ..., 1)` containing + predicted foreground masks. + raymarched.aux["grad_theta"]: A tensor of shape `(B, ..., 3)` containing an + evaluation of a gradient of a signed distance function w.r.t. + input 3D coordinates used to compute the eikonal loss. + raymarched.aux["density_grid"]: A tensor of shape `(B, Hg, Wg, Dg, 1)` + containing a `Hg x Wg x Dg` voxel grid of density values. + ray_bundle: ImplicitronRayBundle object which was used to produce the raymarched + object + image_rgb: A tensor of shape `(B, H, W, 3)` containing ground truth rgb + values. + depth_map: A tensor of shape `(B, Hd, Wd, 1)` containing ground truth depth + values. + fg_probability: A tensor of shape `(B, Hm, Wm, 1)` containing ground truth + foreground masks. + keys_prefix: A common prefix for all keys in the output dictionary + containing all view metrics. + + Returns: + A dictionary `{metric_name_i: metric_value_i}` keyed by the + names of the output metrics `metric_name_i` with their corresponding + values `metric_value_i` represented as 0-dimensional float tensors. + + The calculated metrics are: + rgb_huber: A robust huber loss between `image_pred` and `image`. + rgb_mse: Mean squared error between `image_pred` and `image`. + rgb_psnr: Peak signal-to-noise ratio between `image_pred` and `image`. + rgb_psnr_fg: Peak signal-to-noise ratio between the foreground + region of `image_pred` and `image` as defined by `mask`. + rgb_mse_fg: Mean squared error between the foreground + region of `image_pred` and `image` as defined by `mask`. + mask_neg_iou: (1 - intersection-over-union) between `mask_pred` + and `mask`. + mask_bce: Binary cross entropy between `mask_pred` and `mask`. + mask_beta_prior: A loss enforcing strictly binary values + of `mask_pred`: `log(mask_pred) + log(1-mask_pred)` + depth_abs: Mean per-pixel L1 distance between + `depth_pred` and `depth`. + depth_abs_fg: Mean per-pixel L1 distance between the foreground + region of `depth_pred` and `depth` as defined by `mask`. + eikonal: Eikonal regularizer `(||grad_theta|| - 1)**2`. + density_tv: The Total Variation regularizer of density + values in `density_grid` (sum of L1 distances of values + of all 4-neighbouring cells). + depth_neg_penalty: `min(depth_pred, 0)**2` penalizing negative + predicted depth values. + """ + metrics = self._calculate_stage( + raymarched, + ray_bundle, + image_rgb, + depth_map, + fg_probability, + mask_crop, + keys_prefix, + ) + + if raymarched.prev_stage: + metrics.update( + self( + raymarched.prev_stage, + ray_bundle, + image_rgb, + depth_map, + fg_probability, + mask_crop, + keys_prefix=(keys_prefix + "prev_stage_"), + ) + ) + + return metrics + + def _calculate_stage( + self, + raymarched: RendererOutput, + ray_bundle: ImplicitronRayBundle, + image_rgb: Optional[torch.Tensor] = None, + depth_map: Optional[torch.Tensor] = None, + fg_probability: Optional[torch.Tensor] = None, + mask_crop: Optional[torch.Tensor] = None, + keys_prefix: str = "loss_", + **kwargs, + ) -> Dict[str, Any]: + """ + Calculate metrics for the current stage. + """ + # TODO: extract functions + + # reshape from B x ... x DIM to B x DIM x -1 x 1 + image_rgb_pred, fg_probability_pred, depth_map_pred = [ + _reshape_nongrid_var(x) + for x in [raymarched.features, raymarched.masks, raymarched.depths] + ] + xys = ray_bundle.xys + + # If ray_bundle is packed than we can sample images in padded state to lower + # memory requirements. Instead of having one image for every element in + # ray_bundle we can than have one image per unique sampled camera. + if ray_bundle.is_packed(): + xys, first_idxs, num_inputs = ray_bundle.get_padded_xys() + + # reshape the sampling grid as well + # TODO: we can get rid of the singular dimension here and in _reshape_nongrid_var + # now that we use rend_utils.ndc_grid_sample + xys = xys.reshape(xys.shape[0], -1, 1, 2) + + # closure with the given xys + def sample_full(tensor, mode): + if tensor is None: + return tensor + return rend_utils.ndc_grid_sample(tensor, xys, mode=mode) + + def sample_packed(tensor, mode): + if tensor is None: + return tensor + + # select images that corespond to sampled cameras if raybundle is packed + tensor = tensor[ray_bundle.camera_ids] + if ray_bundle.is_packed(): + # select images that corespond to sampled cameras if raybundle is packed + tensor = tensor[ray_bundle.camera_ids] + result = rend_utils.ndc_grid_sample(tensor, xys, mode=mode) + return padded_to_packed(result, first_idxs, num_inputs, max_size_dim=2)[ + :, :, None + ] # the result is [n_rays_total_training, 3, 1, 1] + + sample = sample_packed if ray_bundle.is_packed() else sample_full + + # eval all results in this size + image_rgb = sample(image_rgb, mode="bilinear") + depth_map = sample(depth_map, mode="nearest") + fg_probability = sample(fg_probability, mode="nearest") + mask_crop = sample(mask_crop, mode="nearest") + if mask_crop is None and image_rgb_pred is not None: + mask_crop = torch.ones_like(image_rgb_pred[:, :1]) + if mask_crop is None and depth_map_pred is not None: + mask_crop = torch.ones_like(depth_map_pred[:, :1]) + + metrics = {} + if image_rgb is not None and image_rgb_pred is not None: + metrics.update( + _rgb_metrics( + image_rgb, + image_rgb_pred, + fg_probability, + fg_probability_pred, + mask_crop, + ) + ) + + if fg_probability_pred is not None: + metrics["mask_beta_prior"] = utils.beta_prior(fg_probability_pred) + if fg_probability is not None and fg_probability_pred is not None: + metrics["mask_neg_iou"] = utils.neg_iou_loss( + fg_probability_pred, fg_probability, mask=mask_crop + ) + metrics["mask_bce"] = utils.calc_bce( + fg_probability_pred, fg_probability, mask=mask_crop + ) + + if depth_map is not None and depth_map_pred is not None: + assert mask_crop is not None + _, abs_ = utils.eval_depth( + depth_map_pred, depth_map, get_best_scale=True, mask=mask_crop, crop=0 + ) + metrics["depth_abs"] = abs_.mean() + + if fg_probability is not None: + mask = fg_probability * mask_crop + _, abs_ = utils.eval_depth( + depth_map_pred, depth_map, get_best_scale=True, mask=mask, crop=0 + ) + metrics["depth_abs_fg"] = abs_.mean() + + # regularizers + grad_theta = raymarched.aux.get("grad_theta") + if grad_theta is not None: + metrics["eikonal"] = _get_eikonal_loss(grad_theta) + + density_grid = raymarched.aux.get("density_grid") + if density_grid is not None: + metrics["density_tv"] = _get_grid_tv_loss(density_grid) + + if depth_map_pred is not None: + metrics["depth_neg_penalty"] = _get_depth_neg_penalty_loss(depth_map_pred) + + if keys_prefix is not None: + metrics = {(keys_prefix + k): v for k, v in metrics.items()} + + return metrics + + +def _rgb_metrics(images, images_pred, masks, masks_pred, masks_crop): + assert masks_crop is not None + if images.shape[1] != images_pred.shape[1]: + raise ValueError( + f"Network output's RGB images had {images_pred.shape[1]} " + f"channels. {images.shape[1]} expected." + ) + rgb_squared = ((images_pred - images) ** 2).mean(dim=1, keepdim=True) + rgb_loss = utils.huber(rgb_squared, scaling=0.03) + crop_mass = masks_crop.sum().clamp(1.0) + results = { + "rgb_huber": (rgb_loss * masks_crop).sum() / crop_mass, + "rgb_mse": (rgb_squared * masks_crop).sum() / crop_mass, + "rgb_psnr": utils.calc_psnr(images_pred, images, mask=masks_crop), + } + if masks is not None: + masks = masks_crop * masks + results["rgb_psnr_fg"] = utils.calc_psnr(images_pred, images, mask=masks) + results["rgb_mse_fg"] = (rgb_squared * masks).sum() / masks.sum().clamp(1.0) + return results + + +def _get_eikonal_loss(grad_theta): + return ((grad_theta.norm(2, dim=1) - 1) ** 2).mean() + + +def _get_grid_tv_loss(grid, log_domain: bool = True, eps: float = 1e-5): + if log_domain: + if (grid <= -eps).any(): + warnings.warn("Grid has negative values; this will produce NaN loss") + grid = torch.log(grid + eps) + + # this is an isotropic version, note that it ignores last rows/cols + return torch.mean( + utils.safe_sqrt( + (grid[..., :-1, :-1, 1:] - grid[..., :-1, :-1, :-1]) ** 2 + + (grid[..., :-1, 1:, :-1] - grid[..., :-1, :-1, :-1]) ** 2 + + (grid[..., 1:, :-1, :-1] - grid[..., :-1, :-1, :-1]) ** 2, + eps=1e-5, + ) + ) + + +def _get_depth_neg_penalty_loss(depth): + neg_penalty = depth.clamp(min=None, max=0.0) ** 2 + return torch.mean(neg_penalty) + + +def _reshape_nongrid_var(x): + if x is None: + return None + + ba, *_, dim = x.shape + return x.reshape(ba, -1, 1, dim).permute(0, 3, 1, 2).contiguous() diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/model_dbir.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/model_dbir.py new file mode 100644 index 0000000000000000000000000000000000000000..496f9ed15d10877d34b4489a032487bf28eba6e4 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/model_dbir.py @@ -0,0 +1,153 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +from typing import Any, Dict, List, Optional, Tuple + +import torch +from pytorch3d.implicitron.dataset.utils import is_known_frame +from pytorch3d.implicitron.tools.config import registry +from pytorch3d.implicitron.tools.point_cloud_utils import ( + get_rgbd_point_cloud, + render_point_cloud_pytorch3d, +) +from pytorch3d.renderer.cameras import CamerasBase +from pytorch3d.structures import Pointclouds + +from .base_model import ImplicitronModelBase, ImplicitronRender +from .renderer.base import EvaluationMode + + +@registry.register +class ModelDBIR(ImplicitronModelBase): + """ + A simple depth-based image rendering model. + + Args: + render_image_width: The width of the rendered rectangular images. + render_image_height: The height of the rendered rectangular images. + bg_color: The color of the background. + max_points: Maximum number of points in the point cloud + formed by unprojecting all source view depths. + If more points are present, they are randomly subsampled + to this number of points without replacement. + """ + + render_image_width: int = 256 + render_image_height: int = 256 + bg_color: Tuple[float, float, float] = (0.0, 0.0, 0.0) + max_points: int = -1 + + # pyre-fixme[14]: `forward` overrides method defined in `ImplicitronModelBase` + # inconsistently. + def forward( + self, + *, # force keyword-only arguments + image_rgb: Optional[torch.Tensor], + camera: CamerasBase, + fg_probability: Optional[torch.Tensor], + mask_crop: Optional[torch.Tensor], + depth_map: Optional[torch.Tensor], + sequence_name: Optional[List[str]], + evaluation_mode: EvaluationMode = EvaluationMode.EVALUATION, + frame_type: List[str], + **kwargs, + ) -> Dict[str, Any]: # TODO: return a namedtuple or dataclass + """ + Given a set of input source cameras images and depth maps, unprojects + all RGBD maps to a colored point cloud and renders into the target views. + + Args: + camera: A batch of `N` PyTorch3D cameras. + image_rgb: A batch of `N` images of shape `(N, 3, H, W)`. + depth_map: A batch of `N` depth maps of shape `(N, 1, H, W)`. + fg_probability: A batch of `N` foreground probability maps + of shape `(N, 1, H, W)`. + frame_type: A list of `N` strings containing frame type indicators + which specify target and source views. + + Returns: + preds: A dict with the following fields: + implicitron_render: The rendered colors, depth and mask + of the target views. + point_cloud: The point cloud of the scene. It's renders are + stored in `implicitron_render`. + """ + + if image_rgb is None: + raise ValueError("ModelDBIR needs image input") + + if fg_probability is None: + raise ValueError("ModelDBIR needs foreground mask input") + + if depth_map is None: + raise ValueError("ModelDBIR needs depth map input") + + is_known = is_known_frame(frame_type) + is_known_idx = torch.where(is_known)[0] + + mask_fg = (fg_probability > 0.5).type_as(image_rgb) + + point_cloud = get_rgbd_point_cloud( + # pyre-fixme[6]: For 1st param expected `Union[List[int], int, + # LongTensor]` but got `Tensor`. + camera[is_known_idx], + image_rgb[is_known_idx], + depth_map[is_known_idx], + mask_fg[is_known_idx], + ) + + pcl_size = point_cloud.num_points_per_cloud().item() + if (self.max_points > 0) and (pcl_size > self.max_points): + # pyre-fixme[6]: For 1st param expected `int` but got `Union[bool, + # float, int]`. + prm = torch.randperm(pcl_size)[: self.max_points] + point_cloud = Pointclouds( + point_cloud.points_padded()[:, prm, :], + # pyre-fixme[16]: Optional type has no attribute `__getitem__`. + features=point_cloud.features_padded()[:, prm, :], + ) + + is_target_idx = torch.where(~is_known)[0] + + depth_render, image_render, mask_render = [], [], [] + + # render into target frames in a for loop to save memory + for tgt_idx in is_target_idx: + _image_render, _mask_render, _depth_render = render_point_cloud_pytorch3d( + camera[int(tgt_idx)], + point_cloud, + render_size=(self.render_image_height, self.render_image_width), + point_radius=1e-2, + topk=10, + bg_color=self.bg_color, + ) + _image_render = _image_render.clamp(0.0, 1.0) + # the mask is the set of pixels with opacity bigger than eps + _mask_render = (_mask_render > 1e-4).float() + + depth_render.append(_depth_render) + image_render.append(_image_render) + mask_render.append(_mask_render) + + implicitron_render = ImplicitronRender( + **{ + k: torch.cat(v, dim=0) + for k, v in zip( + ["depth_render", "image_render", "mask_render"], + [depth_render, image_render, mask_render], + ) + } + ) + + preds = { + "implicitron_render": implicitron_render, + "point_cloud": point_cloud, + } + + return preds diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/overfit_model.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/overfit_model.py new file mode 100644 index 0000000000000000000000000000000000000000..f1518e778948250a809f7ee9db177032249f0fe7 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/overfit_model.py @@ -0,0 +1,671 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +# Note: The #noqa comments below are for unused imports of pluggable implementations +# which are part of implicitron. They ensure that the registry is prepopulated. + +import functools +import logging +from dataclasses import field +from typing import Any, Callable, Dict, List, Optional, Tuple, TYPE_CHECKING, Union + +import torch +from omegaconf import DictConfig + +from pytorch3d.implicitron.models.base_model import ( + ImplicitronModelBase, + ImplicitronRender, +) +from pytorch3d.implicitron.models.global_encoder.global_encoder import GlobalEncoderBase +from pytorch3d.implicitron.models.implicit_function.base import ImplicitFunctionBase +from pytorch3d.implicitron.models.metrics import ( + RegularizationMetricsBase, + ViewMetricsBase, +) + +from pytorch3d.implicitron.models.renderer.base import ( + BaseRenderer, + EvaluationMode, + ImplicitronRayBundle, + RendererOutput, + RenderSamplingMode, +) +from pytorch3d.implicitron.models.renderer.ray_sampler import RaySamplerBase +from pytorch3d.implicitron.models.utils import ( + apply_chunked, + chunk_generator, + log_loss_weights, + preprocess_input, + weighted_sum_losses, +) +from pytorch3d.implicitron.tools import vis_utils +from pytorch3d.implicitron.tools.config import ( + expand_args_fields, + registry, + run_auto_creation, +) + +from pytorch3d.implicitron.tools.rasterize_mc import rasterize_sparse_ray_bundle +from pytorch3d.renderer import utils as rend_utils +from pytorch3d.renderer.cameras import CamerasBase + + +if TYPE_CHECKING: + from visdom import Visdom +logger = logging.getLogger(__name__) + +IMPLICIT_FUNCTION_ARGS_TO_REMOVE: List[str] = [ + "feature_vector_size", + "encoding_dim", + "latent_dim", + "color_dim", +] + + +@registry.register +class OverfitModel(ImplicitronModelBase): # pyre-ignore: 13 + """ + OverfitModel is a wrapper for the neural implicit + rendering and reconstruction pipeline which consists + of the following sequence of 4 steps: + + + (1) Ray Sampling + ------------------ + Rays are sampled from an image grid based on the target view(s). + │ + ▼ + (2) Implicit Function Evaluation + ------------------ + Evaluate the implicit function(s) at the sampled ray points + (also optionally pass in a global encoding from global_encoder). + │ + ▼ + (3) Rendering + ------------------ + Render the image into the target cameras by raymarching along + the sampled rays and aggregating the colors and densities + output by the implicit function in (2). + │ + ▼ + (4) Loss Computation + ------------------ + Compute losses based on the predicted target image(s). + + + The `forward` function of OverfitModel executes + this sequence of steps. Currently, steps 1, 2, 3 + can be customized by intializing a subclass of the appropriate + base class and adding the newly created module to the registry. + Please see https://github.com/facebookresearch/pytorch3d/blob/main/projects/implicitron_trainer/README.md#custom-plugins + for more details on how to create and register a custom component. + + In the config .yaml files for experiments, the parameters below are + contained in the + `model_factory_ImplicitronModelFactory_args.model_OverfitModel_args` + node. As OverfitModel derives from ReplaceableBase, the input arguments are + parsed by the run_auto_creation function to initialize the + necessary member modules. Please see implicitron_trainer/README.md + for more details on this process. + + Args: + mask_images: Whether or not to mask the RGB image background given the + foreground mask (the `fg_probability` argument of `GenericModel.forward`) + mask_depths: Whether or not to mask the depth image background given the + foreground mask (the `fg_probability` argument of `GenericModel.forward`) + render_image_width: Width of the output image to render + render_image_height: Height of the output image to render + mask_threshold: If greater than 0.0, the foreground mask is + thresholded by this value before being applied to the RGB/Depth images + output_rasterized_mc: If True, visualize the Monte-Carlo pixel renders by + splatting onto an image grid. Default: False. + bg_color: RGB values for setting the background color of input image + if mask_images=True. Defaults to (0.0, 0.0, 0.0). Each renderer has its own + way to determine the background color of its output, unrelated to this. + chunk_size_grid: The total number of points which can be rendered + per chunk. This is used to compute the number of rays used + per chunk when the chunked version of the renderer is used (in order + to fit rendering on all rays in memory) + render_features_dimensions: The number of output features to render. + Defaults to 3, corresponding to RGB images. + sampling_mode_training: The sampling method to use during training. Must be + a value from the RenderSamplingMode Enum. + sampling_mode_evaluation: Same as above but for evaluation. + global_encoder_class_type: The name of the class to use for global_encoder, + which must be available in the registry. Or `None` to disable global encoder. + global_encoder: An instance of `GlobalEncoder`. This is used to generate an encoding + of the image (referred to as the global_code) that can be used to model aspects of + the scene such as multiple objects or morphing objects. It is up to the implicit + function definition how to use it, but the most typical way is to broadcast and + concatenate to the other inputs for the implicit function. + raysampler_class_type: The name of the raysampler class which is available + in the global registry. + raysampler: An instance of RaySampler which is used to emit + rays from the target view(s). + renderer_class_type: The name of the renderer class which is available in the global + registry. + renderer: A renderer class which inherits from BaseRenderer. This is used to + generate the images from the target view(s). + share_implicit_function_across_passes: If set to True + coarse_implicit_function is automatically set as implicit_function + (coarse_implicit_function=implicit_funciton). The + implicit_functions are then run sequentially during the rendering. + implicit_function_class_type: The type of implicit function to use which + is available in the global registry. + implicit_function: An instance of ImplicitFunctionBase. + coarse_implicit_function_class_type: The type of implicit function to use which + is available in the global registry. + coarse_implicit_function: An instance of ImplicitFunctionBase. + If set and `share_implicit_function_across_passes` is set to False, + coarse_implicit_function is instantiated on itself. It + is then used as the second pass during the rendering. + If set to None, we only do a single pass with implicit_function. + view_metrics: An instance of ViewMetricsBase used to compute loss terms which + are independent of the model's parameters. + view_metrics_class_type: The type of view metrics to use, must be available in + the global registry. + regularization_metrics: An instance of RegularizationMetricsBase used to compute + regularization terms which can depend on the model's parameters. + regularization_metrics_class_type: The type of regularization metrics to use, + must be available in the global registry. + loss_weights: A dictionary with a {loss_name: weight} mapping; see documentation + for `ViewMetrics` class for available loss functions. + log_vars: A list of variable names which should be logged. + The names should correspond to a subset of the keys of the + dict `preds` output by the `forward` function. + """ # noqa: B950 + + mask_images: bool = True + mask_depths: bool = True + render_image_width: int = 400 + render_image_height: int = 400 + mask_threshold: float = 0.5 + output_rasterized_mc: bool = False + bg_color: Tuple[float, float, float] = (0.0, 0.0, 0.0) + chunk_size_grid: int = 4096 + render_features_dimensions: int = 3 + tqdm_trigger_threshold: int = 16 + + n_train_target_views: int = 1 + sampling_mode_training: str = "mask_sample" + sampling_mode_evaluation: str = "full_grid" + + # ---- global encoder settings + global_encoder_class_type: Optional[str] = None + global_encoder: Optional[GlobalEncoderBase] + + # ---- raysampler + raysampler_class_type: str = "AdaptiveRaySampler" + raysampler: RaySamplerBase + + # ---- renderer configs + renderer_class_type: str = "MultiPassEmissionAbsorptionRenderer" + renderer: BaseRenderer + + # ---- implicit function settings + share_implicit_function_across_passes: bool = False + implicit_function_class_type: str = "NeuralRadianceFieldImplicitFunction" + implicit_function: ImplicitFunctionBase + coarse_implicit_function_class_type: Optional[str] = None + coarse_implicit_function: Optional[ImplicitFunctionBase] + + # ----- metrics + view_metrics: ViewMetricsBase + view_metrics_class_type: str = "ViewMetrics" + + regularization_metrics: RegularizationMetricsBase + regularization_metrics_class_type: str = "RegularizationMetrics" + + # ---- loss weights + loss_weights: Dict[str, float] = field( + default_factory=lambda: { + "loss_rgb_mse": 1.0, + "loss_prev_stage_rgb_mse": 1.0, + "loss_mask_bce": 0.0, + "loss_prev_stage_mask_bce": 0.0, + } + ) + + # ---- variables to be logged (logger automatically ignores if not computed) + log_vars: List[str] = field( + default_factory=lambda: [ + "loss_rgb_psnr_fg", + "loss_rgb_psnr", + "loss_rgb_mse", + "loss_rgb_huber", + "loss_depth_abs", + "loss_depth_abs_fg", + "loss_mask_neg_iou", + "loss_mask_bce", + "loss_mask_beta_prior", + "loss_eikonal", + "loss_density_tv", + "loss_depth_neg_penalty", + "loss_autodecoder_norm", + # metrics that are only logged in 2+stage renderes + "loss_prev_stage_rgb_mse", + "loss_prev_stage_rgb_psnr_fg", + "loss_prev_stage_rgb_psnr", + "loss_prev_stage_mask_bce", + # basic metrics + "objective", + "epoch", + "sec/it", + ] + ) + + @classmethod + def pre_expand(cls) -> None: + # use try/finally to bypass cinder's lazy imports + try: + from pytorch3d.implicitron.models.implicit_function.idr_feature_field import ( # noqa: F401, B950 + IdrFeatureField, + ) + from pytorch3d.implicitron.models.implicit_function.neural_radiance_field import ( # noqa: F401, B950 + NeuralRadianceFieldImplicitFunction, + ) + from pytorch3d.implicitron.models.implicit_function.scene_representation_networks import ( # noqa: F401, B950 + SRNImplicitFunction, + ) + from pytorch3d.implicitron.models.renderer.lstm_renderer import ( # noqa: F401 + LSTMRenderer, + ) + from pytorch3d.implicitron.models.renderer.multipass_ea import ( # noqa: F401 + MultiPassEmissionAbsorptionRenderer, + ) + from pytorch3d.implicitron.models.renderer.sdf_renderer import ( # noqa: F401 + SignedDistanceFunctionRenderer, + ) + finally: + pass + + def __post_init__(self): + # The attribute will be filled by run_auto_creation + run_auto_creation(self) + log_loss_weights(self.loss_weights, logger) + # We need to set it here since run_auto_creation + # will create coarse_implicit_function before implicit_function + if self.share_implicit_function_across_passes: + self.coarse_implicit_function = self.implicit_function + + def forward( + self, + *, # force keyword-only arguments + image_rgb: Optional[torch.Tensor], + camera: CamerasBase, + fg_probability: Optional[torch.Tensor] = None, + mask_crop: Optional[torch.Tensor] = None, + depth_map: Optional[torch.Tensor] = None, + sequence_name: Optional[List[str]] = None, + frame_timestamp: Optional[torch.Tensor] = None, + evaluation_mode: EvaluationMode = EvaluationMode.EVALUATION, + **kwargs, + ) -> Dict[str, Any]: + """ + Args: + image_rgb: A tensor of shape `(B, 3, H, W)` containing a batch of rgb images; + the first `min(B, n_train_target_views)` images are considered targets and + are used to supervise the renders; the rest corresponding to the source + viewpoints from which features will be extracted. + camera: An instance of CamerasBase containing a batch of `B` cameras corresponding + to the viewpoints of target images, from which the rays will be sampled, + and source images, which will be used for intersecting with target rays. + fg_probability: A tensor of shape `(B, 1, H, W)` containing a batch of + foreground masks. + mask_crop: A binary tensor of shape `(B, 1, H, W)` deonting valid + regions in the input images (i.e. regions that do not correspond + to, e.g., zero-padding). When the `RaySampler`'s sampling mode is set to + "mask_sample", rays will be sampled in the non zero regions. + depth_map: A tensor of shape `(B, 1, H, W)` containing a batch of depth maps. + sequence_name: A list of `B` strings corresponding to the sequence names + from which images `image_rgb` were extracted. They are used to match + target frames with relevant source frames. + frame_timestamp: Optionally a tensor of shape `(B,)` containing a batch + of frame timestamps. + evaluation_mode: one of EvaluationMode.TRAINING or + EvaluationMode.EVALUATION which determines the settings used for + rendering. + + Returns: + preds: A dictionary containing all outputs of the forward pass including the + rendered images, depths, masks, losses and other metrics. + """ + image_rgb, fg_probability, depth_map = preprocess_input( + image_rgb, + fg_probability, + depth_map, + self.mask_images, + self.mask_depths, + self.mask_threshold, + self.bg_color, + ) + + # Determine the used ray sampling mode. + sampling_mode = RenderSamplingMode( + self.sampling_mode_training + if evaluation_mode == EvaluationMode.TRAINING + else self.sampling_mode_evaluation + ) + + # (1) Sample rendering rays with the ray sampler. + # pyre-ignore[29] + ray_bundle: ImplicitronRayBundle = self.raysampler( + camera, + evaluation_mode, + mask=( + mask_crop + if mask_crop is not None + and sampling_mode == RenderSamplingMode.MASK_SAMPLE + else None + ), + ) + + inputs_to_be_chunked = {} + if fg_probability is not None and self.renderer.requires_object_mask(): + sampled_fb_prob = rend_utils.ndc_grid_sample( + fg_probability, ray_bundle.xys, mode="nearest" + ) + inputs_to_be_chunked["object_mask"] = sampled_fb_prob > 0.5 + + # (2)-(3) Implicit function evaluation and Rendering + implicit_functions: List[Union[Callable, ImplicitFunctionBase]] = [ + self.implicit_function + ] + if self.coarse_implicit_function is not None: + implicit_functions = [self.coarse_implicit_function, self.implicit_function] + + if self.global_encoder is not None: + global_code = self.global_encoder( # pyre-fixme[29] + sequence_name=sequence_name, + frame_timestamp=frame_timestamp, + ) + implicit_functions = [ + ( + functools.partial(implicit_function, global_code=global_code) + if isinstance(implicit_function, Callable) + else functools.partial( + implicit_function.forward, global_code=global_code + ) + ) + for implicit_function in implicit_functions + ] + rendered = self._render( + ray_bundle=ray_bundle, + sampling_mode=sampling_mode, + evaluation_mode=evaluation_mode, + implicit_functions=implicit_functions, + inputs_to_be_chunked=inputs_to_be_chunked, + ) + + # A dict to store losses as well as rendering results. + preds: Dict[str, Any] = self.view_metrics( + results={}, + raymarched=rendered, + ray_bundle=ray_bundle, + image_rgb=image_rgb, + depth_map=depth_map, + fg_probability=fg_probability, + mask_crop=mask_crop, + ) + + preds.update( + self.regularization_metrics( + results=preds, + model=self, + ) + ) + + if sampling_mode == RenderSamplingMode.MASK_SAMPLE: + if self.output_rasterized_mc: + # Visualize the monte-carlo pixel renders by splatting onto + # an image grid. + ( + preds["images_render"], + preds["depths_render"], + preds["masks_render"], + ) = rasterize_sparse_ray_bundle( + ray_bundle, + rendered.features, + (self.render_image_height, self.render_image_width), + rendered.depths, + masks=rendered.masks, + ) + elif sampling_mode == RenderSamplingMode.FULL_GRID: + preds["images_render"] = rendered.features.permute(0, 3, 1, 2) + preds["depths_render"] = rendered.depths.permute(0, 3, 1, 2) + preds["masks_render"] = rendered.masks.permute(0, 3, 1, 2) + + preds["implicitron_render"] = ImplicitronRender( + image_render=preds["images_render"], + depth_render=preds["depths_render"], + mask_render=preds["masks_render"], + ) + else: + raise AssertionError("Unreachable state") + + # (4) Compute losses + # finally get the optimization objective using self.loss_weights + objective = self._get_objective(preds) + if objective is not None: + preds["objective"] = objective + + return preds + + def _get_objective(self, preds: Dict[str, torch.Tensor]) -> Optional[torch.Tensor]: + """ + A helper function to compute the overall loss as the dot product + of individual loss functions with the corresponding weights. + """ + return weighted_sum_losses(preds, self.loss_weights) + + def visualize( + self, + viz: Optional["Visdom"], + visdom_env_imgs: str, + preds: Dict[str, Any], + prefix: str, + ) -> None: + """ + Helper function to visualize the predictions generated + in the forward pass. + + Args: + viz: Visdom connection object + visdom_env_imgs: name of visdom environment for the images. + preds: predictions dict like returned by forward() + prefix: prepended to the names of images + """ + if viz is None or not viz.check_connection(): + logger.info("no visdom server! -> skipping batch vis") + return + + idx_image = 0 + title = f"{prefix}_im{idx_image}" + + vis_utils.visualize_basics(viz, preds, visdom_env_imgs, title=title) + + def _render( + self, + *, + ray_bundle: ImplicitronRayBundle, + inputs_to_be_chunked: Dict[str, torch.Tensor], + sampling_mode: RenderSamplingMode, + **kwargs, + ) -> RendererOutput: + """ + Args: + ray_bundle: A `ImplicitronRayBundle` object containing the parametrizations of the + sampled rendering rays. + inputs_to_be_chunked: A collection of tensor of shape `(B, _, H, W)`. E.g. + SignedDistanceFunctionRenderer requires "object_mask", shape + (B, 1, H, W), the silhouette of the object in the image. When + chunking, they are passed to the renderer as shape + `(B, _, chunksize)`. + sampling_mode: The sampling method to use. Must be a value from the + RenderSamplingMode Enum. + + Returns: + An instance of RendererOutput + """ + if sampling_mode == RenderSamplingMode.FULL_GRID and self.chunk_size_grid > 0: + return apply_chunked( + self.renderer, + chunk_generator( + self.chunk_size_grid, + ray_bundle, + inputs_to_be_chunked, + self.tqdm_trigger_threshold, + **kwargs, + ), + lambda batch: torch.cat(batch, dim=1).reshape( + *ray_bundle.lengths.shape[:-1], -1 + ), + ) + else: + # pyre-fixme[29]: `BaseRenderer` is not a function. + return self.renderer( + ray_bundle=ray_bundle, + **inputs_to_be_chunked, + **kwargs, + ) + + @classmethod + def raysampler_tweak_args(cls, type, args: DictConfig) -> None: + """ + We don't expose certain fields of the raysampler because we want to set + them from our own members. + """ + del args["sampling_mode_training"] + del args["sampling_mode_evaluation"] + del args["image_width"] + del args["image_height"] + + def create_raysampler(self): + extra_args = { + "sampling_mode_training": self.sampling_mode_training, + "sampling_mode_evaluation": self.sampling_mode_evaluation, + "image_width": self.render_image_width, + "image_height": self.render_image_height, + } + raysampler_args = getattr( + self, "raysampler_" + self.raysampler_class_type + "_args" + ) + self.raysampler = registry.get(RaySamplerBase, self.raysampler_class_type)( + **raysampler_args, **extra_args + ) + + @classmethod + def renderer_tweak_args(cls, type, args: DictConfig) -> None: + """ + We don't expose certain fields of the renderer because we want to set + them based on other inputs. + """ + args.pop("render_features_dimensions", None) + args.pop("object_bounding_sphere", None) + + def create_renderer(self): + extra_args = {} + + if self.renderer_class_type == "SignedDistanceFunctionRenderer": + extra_args["render_features_dimensions"] = self.render_features_dimensions + if not hasattr(self.raysampler, "scene_extent"): + raise ValueError( + "SignedDistanceFunctionRenderer requires" + + " a raysampler that defines the 'scene_extent' field" + + " (this field is supported by, e.g., the adaptive raysampler - " + + " self.raysampler_class_type='AdaptiveRaySampler')." + ) + extra_args["object_bounding_sphere"] = self.raysampler.scene_extent + + renderer_args = getattr(self, "renderer_" + self.renderer_class_type + "_args") + self.renderer = registry.get(BaseRenderer, self.renderer_class_type)( + **renderer_args, **extra_args + ) + + @classmethod + def implicit_function_tweak_args(cls, type, args: DictConfig) -> None: + """ + We don't expose certain implicit_function fields because we want to set + them based on other inputs. + """ + for arg in IMPLICIT_FUNCTION_ARGS_TO_REMOVE: + args.pop(arg, None) + + @classmethod + def coarse_implicit_function_tweak_args(cls, type, args: DictConfig) -> None: + """ + We don't expose certain implicit_function fields because we want to set + them based on other inputs. + """ + for arg in IMPLICIT_FUNCTION_ARGS_TO_REMOVE: + args.pop(arg, None) + + def _create_extra_args_for_implicit_function(self) -> Dict[str, Any]: + extra_args = {} + global_encoder_dim = ( + 0 if self.global_encoder is None else self.global_encoder.get_encoding_dim() + ) + if self.implicit_function_class_type in ( + "NeuralRadianceFieldImplicitFunction", + "NeRFormerImplicitFunction", + ): + extra_args["latent_dim"] = global_encoder_dim + extra_args["color_dim"] = self.render_features_dimensions + + if self.implicit_function_class_type == "IdrFeatureField": + extra_args["feature_work_size"] = global_encoder_dim + extra_args["feature_vector_size"] = self.render_features_dimensions + + if self.implicit_function_class_type == "SRNImplicitFunction": + extra_args["latent_dim"] = global_encoder_dim + return extra_args + + def create_implicit_function(self) -> None: + implicit_function_type = registry.get( + ImplicitFunctionBase, self.implicit_function_class_type + ) + expand_args_fields(implicit_function_type) + + config_name = f"implicit_function_{self.implicit_function_class_type}_args" + config = getattr(self, config_name, None) + if config is None: + raise ValueError(f"{config_name} not present") + + extra_args = self._create_extra_args_for_implicit_function() + self.implicit_function = implicit_function_type(**config, **extra_args) + + def create_coarse_implicit_function(self) -> None: + # If coarse_implicit_function_class_type has been defined + # then we init a module based on its arguments + if ( + self.coarse_implicit_function_class_type is not None + and not self.share_implicit_function_across_passes + ): + config_name = "coarse_implicit_function_{0}_args".format( + self.coarse_implicit_function_class_type + ) + config = getattr(self, config_name, {}) + + implicit_function_type = registry.get( + ImplicitFunctionBase, + # pyre-ignore: config is None allow to check if this is None. + self.coarse_implicit_function_class_type, + ) + expand_args_fields(implicit_function_type) + + extra_args = self._create_extra_args_for_implicit_function() + self.coarse_implicit_function = implicit_function_type( + **config, **extra_args + ) + elif self.share_implicit_function_across_passes: + # Since coarse_implicit_function is initialised before + # implicit_function we handle this case in the post_init. + pass + else: + self.coarse_implicit_function = None diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/__init__.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ac1a72bde66f104691245d2de4e83c6863718d5 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/base.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/base.py new file mode 100644 index 0000000000000000000000000000000000000000..a92cb8f173a08f3f534407e640ed041b8eb68dcf --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/base.py @@ -0,0 +1,426 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Dict, List, Optional, Tuple + +import torch +from pytorch3d.implicitron.tools.config import ReplaceableBase +from pytorch3d.ops import packed_to_padded +from pytorch3d.renderer.implicit.utils import ray_bundle_variables_to_ray_points + + +class EvaluationMode(Enum): + TRAINING = "training" + EVALUATION = "evaluation" + + +class RenderSamplingMode(Enum): + MASK_SAMPLE = "mask_sample" + FULL_GRID = "full_grid" + + +class ImplicitronRayBundle: + """ + Parametrizes points along projection rays by storing ray `origins`, + `directions` vectors and `lengths` at which the ray-points are sampled. + Furthermore, the xy-locations (`xys`) of the ray pixels are stored as well. + Note that `directions` don't have to be normalized; they define unit vectors + in the respective 1D coordinate systems; see documentation for + :func:`ray_bundle_to_ray_points` for the conversion formula. + + Ray bundle may represent rays from multiple cameras. In that case, cameras + are stored in the packed form (i.e. rays from the same camera are stored in + the consecutive elements). The following indices will be set: + camera_ids: A tensor of shape (N, ) which indicates which camera + was used to sample the rays. `N` is the number of different + sampled cameras. + camera_counts: A tensor of shape (N, ) which how many times the + coresponding camera in `camera_ids` was sampled. + `sum(camera_counts) == minibatch`, where `minibatch = origins.shape[0]`. + + Attributes: + origins: A tensor of shape `(..., 3)` denoting the + origins of the sampling rays in world coords. + directions: A tensor of shape `(..., 3)` containing the direction + vectors of sampling rays in world coords. They don't have to be normalized; + they define unit vectors in the respective 1D coordinate systems; see + documentation for :func:`ray_bundle_to_ray_points` for the conversion formula. + lengths: A tensor of shape `(..., num_points_per_ray)` + containing the lengths at which the rays are sampled. + xys: A tensor of shape `(..., 2)`, the xy-locations (`xys`) of the ray pixels + camera_ids: An optional tensor of shape (N, ) which indicates which camera + was used to sample the rays. `N` is the number of unique sampled cameras. + camera_counts: An optional tensor of shape (N, ) indicates how many times the + coresponding camera in `camera_ids` was sampled. + `sum(camera_counts)==total_number_of_rays`. + bins: An optional tensor of shape `(..., num_points_per_ray + 1)` + containing the bins at which the rays are sampled. In this case + lengths should be equal to the midpoints of bins `(..., num_points_per_ray)`. + pixel_radii_2d: An optional tensor of shape `(..., 1)` + base radii of the conical frustums. + + Raises: + ValueError: If either bins or lengths are not provided. + ValueError: If bins is provided and the last dim is inferior or equal to 1. + """ + + def __init__( + self, + origins: torch.Tensor, + directions: torch.Tensor, + lengths: Optional[torch.Tensor], + xys: torch.Tensor, + camera_ids: Optional[torch.LongTensor] = None, + camera_counts: Optional[torch.LongTensor] = None, + bins: Optional[torch.Tensor] = None, + pixel_radii_2d: Optional[torch.Tensor] = None, + ): + if bins is not None and bins.shape[-1] <= 1: + raise ValueError( + "The last dim of bins must be at least superior or equal to 2." + ) + + if bins is None and lengths is None: + raise ValueError( + "Please set either bins or lengths to initialize an ImplicitronRayBundle." + ) + + self.origins = origins + self.directions = directions + self._lengths = lengths if bins is None else None + self.xys = xys + self.bins = bins + self.pixel_radii_2d = pixel_radii_2d + self.camera_ids = camera_ids + self.camera_counts = camera_counts + + @property + def lengths(self) -> torch.Tensor: + if self.bins is not None: + # equivalent to: 0.5 * (bins[..., 1:] + bins[..., :-1]) but more efficient + # pyre-ignore + return torch.lerp(self.bins[..., :-1], self.bins[..., 1:], 0.5) + return self._lengths + + @lengths.setter + def lengths(self, value): + if self.bins is not None: + raise ValueError( + "If the bins attribute is not None you cannot set the lengths attribute." + ) + else: + self._lengths = value + + def float_(self) -> None: + """Moves the tensors to float dtype in place + (helpful for mixed-precision tensors). + """ + self.origins = self.origins.float() + self.directions = self.directions.float() + self._lengths = self._lengths.float() if self._lengths is not None else None + self.xys = self.xys.float() + self.bins = self.bins.float() if self.bins is not None else None + self.pixel_radii_2d = ( + self.pixel_radii_2d.float() if self.pixel_radii_2d is not None else None + ) + + def is_packed(self) -> bool: + """ + Returns whether the ImplicitronRayBundle carries data in packed state + """ + return self.camera_ids is not None and self.camera_counts is not None + + def get_padded_xys(self) -> Tuple[torch.Tensor, torch.LongTensor, int]: + """ + For a packed ray bundle, returns padded rays. Assumes the input bundle is packed + (i.e. `camera_ids` and `camera_counts` are set). + + Returns: + - xys: Tensor of shape (N, max_size, ...) containing the padded + representation of the pixel coordinated; + where max_size is max of `camera_counts`. The values for camera id `i` + will be copied to `xys[i, :]`, with zeros padding out the extra inputs. + - first_idxs: cumulative sum of `camera_counts` defininf the boundaries + between cameras in the packed representation + - num_inputs: the number of cameras in the bundle. + """ + if not self.is_packed(): + raise ValueError("get_padded_xys can be called only on a packed bundle") + + camera_counts = self.camera_counts + assert camera_counts is not None + + cumsum = torch.cumsum(camera_counts, dim=0, dtype=torch.long) + first_idxs = torch.cat( + (camera_counts.new_zeros((1,), dtype=torch.long), cumsum[:-1]) + ) + num_inputs = camera_counts.sum().item() + max_size = torch.max(camera_counts).item() + xys = packed_to_padded(self.xys, first_idxs, max_size) + # pyre-ignore [7] pytorch typeshed inaccuracy + return xys, first_idxs, num_inputs + + +@dataclass +class RendererOutput: + """ + A structure for storing the output of a renderer. + + Args: + features: rendered features (usually RGB colors), (B, ..., C) tensor. + depth: rendered ray-termination depth map, in NDC coordinates, (B, ..., 1) tensor. + mask: rendered object mask, values in [0, 1], (B, ..., 1) tensor. + prev_stage: for multi-pass renderers (e.g. in NeRF), + a reference to the output of the previous stage. + normals: surface normals, for renderers that estimate them; (B, ..., 3) tensor. + points: ray-termination points in the world coordinates, (B, ..., 3) tensor. + aux: dict for implementation-specific renderer outputs. + """ + + features: torch.Tensor + depths: torch.Tensor + masks: torch.Tensor + prev_stage: Optional[RendererOutput] = None + normals: Optional[torch.Tensor] = None + points: Optional[torch.Tensor] = None # TODO: redundant with depths + weights: Optional[torch.Tensor] = None + aux: Dict[str, Any] = field(default_factory=lambda: {}) + + +class ImplicitFunctionWrapper(torch.nn.Module): + def __init__(self, fn: torch.nn.Module): + super().__init__() + self._fn = fn + self.bound_args = {} + + def bind_args(self, **bound_args): + self.bound_args = bound_args + self._fn.on_bind_args() + + def unbind_args(self): + self.bound_args = {} + + def forward(self, *args, **kwargs): + return self._fn(*args, **{**kwargs, **self.bound_args}) + + +class BaseRenderer(ABC, ReplaceableBase): + """ + Base class for all Renderer implementations. + """ + + def requires_object_mask(self) -> bool: + """ + Whether `forward` needs the object_mask. + """ + return False + + @abstractmethod + def forward( + self, + ray_bundle: ImplicitronRayBundle, + implicit_functions: List[ImplicitFunctionWrapper], + evaluation_mode: EvaluationMode = EvaluationMode.EVALUATION, + **kwargs, + ) -> RendererOutput: + """ + Each Renderer should implement its own forward function + that returns an instance of RendererOutput. + + Args: + ray_bundle: An ImplicitronRayBundle object containing the following variables: + origins: A tensor of shape (minibatch, ..., 3) denoting + the origins of the rendering rays. + directions: A tensor of shape (minibatch, ..., 3) + containing the direction vectors of rendering rays. + lengths: A tensor of shape + (minibatch, ..., num_points_per_ray)containing the + lengths at which the ray points are sampled. + The coordinates of the points on the rays are thus computed + as `origins + lengths * directions`. + xys: A tensor of shape + (minibatch, ..., 2) containing the + xy locations of each ray's pixel in the NDC screen space. + camera_ids: A tensor of shape (N, ) which indicates which camera + was used to sample the rays. `N` is the number of different + sampled cameras. + camera_counts: A tensor of shape (N, ) which how many times the + coresponding camera in `camera_ids` was sampled. + `sum(camera_counts)==minibatch` + implicit_functions: List of ImplicitFunctionWrappers which define the + implicit function methods to be used. Most Renderers only allow + a single implicit function. Currently, only the + MultiPassEmissionAbsorptionRenderer allows specifying mulitple + values in the list. + evaluation_mode: one of EvaluationMode.TRAINING or + EvaluationMode.EVALUATION which determines the settings used for + rendering. + **kwargs: In addition to the name args, custom keyword args can be specified. + For example in the SignedDistanceFunctionRenderer, an object_mask is + required which needs to be passed via the kwargs. + + Returns: + instance of RendererOutput + """ + pass + + +def compute_3d_diagonal_covariance_gaussian( + rays_directions: torch.Tensor, + rays_dir_variance: torch.Tensor, + radii_variance: torch.Tensor, + eps: float = 1e-6, +) -> torch.Tensor: + """ + Transform the variances (rays_dir_variance, radii_variance) of the gaussians from + the coordinate frame of the conical frustum to 3D world coordinates. + + It follows the equation 16 of `MIP-NeRF `_ + + Args: + rays_directions: A tensor of shape `(..., 3)` + rays_dir_variance: A tensor of shape `(..., num_intervals)` representing + the variance of the conical frustum with respect to the rays direction. + radii_variance: A tensor of shape `(..., num_intervals)` representing + the variance of the conical frustum with respect to its radius. + eps: a small number to prevent division by zero. + + Returns: + A tensor of shape `(..., num_intervals, 3)` containing the diagonal + of the covariance matrix. + """ + d_outer_diag = torch.pow(rays_directions, 2) + dir_mag_sq = torch.clamp(torch.sum(d_outer_diag, dim=-1, keepdim=True), min=eps) + + null_outer_diag = 1 - d_outer_diag / dir_mag_sq + ray_dir_cov_diag = rays_dir_variance[..., None] * d_outer_diag[..., None, :] + xy_cov_diag = radii_variance[..., None] * null_outer_diag[..., None, :] + return ray_dir_cov_diag + xy_cov_diag + + +def approximate_conical_frustum_as_gaussians( + bins: torch.Tensor, radii: torch.Tensor +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Approximates a conical frustum as two Gaussian distributions. + + The Gaussian distributions are characterized by + three values: + + - rays_dir_mean: mean along the rays direction + (defined as t in the parametric representation of a cone). + - rays_dir_variance: the variance of the conical frustum along the rays direction. + - radii_variance: variance of the conical frustum with respect to its radius. + + + The computation is stable and follows equation 7 + of `MIP-NeRF `_. + + For more information on how the mean and variances are computed + refers to the appendix of the paper. + + Args: + bins: A tensor of shape `(..., num_points_per_ray + 1)` + containing the bins at which the rays are sampled. + `bin[..., t]` and `bin[..., t+1]` represent respectively + the left and right coordinates of the interval. + t0: A tensor of shape `(..., num_points_per_ray)` + containing the left coordinates of the intervals + on which the rays are sampled. + t1: A tensor of shape `(..., num_points_per_ray)` + containing the rights coordinates of the intervals + on which the rays are sampled. + radii: A tensor of shape `(..., 1)` + base radii of the conical frustums. + + Returns: + rays_dir_mean: A tensor of shape `(..., num_intervals)` representing + the mean along the rays direction + (t in the parametric represention of the cone) + rays_dir_variance: A tensor of shape `(..., num_intervals)` representing + the variance of the conical frustum along the rays + (t in the parametric represention of the cone). + radii_variance: A tensor of shape `(..., num_intervals)` representing + the variance of the conical frustum with respect to its radius. + """ + t_mu = torch.lerp(bins[..., 1:], bins[..., :-1], 0.5) + t_delta = torch.diff(bins, dim=-1) / 2 + + t_mu_pow2 = torch.pow(t_mu, 2) + t_delta_pow2 = torch.pow(t_delta, 2) + t_delta_pow4 = torch.pow(t_delta, 4) + + den = 3 * t_mu_pow2 + t_delta_pow2 + + # mean along the rays direction + rays_dir_mean = t_mu + 2 * t_mu * t_delta_pow2 / den + + # Variance of the conical frustum with along the rays directions + rays_dir_variance = t_delta_pow2 / 3 - (4 / 15) * ( + t_delta_pow4 * (12 * t_mu_pow2 - t_delta_pow2) / torch.pow(den, 2) + ) + + # Variance of the conical frustum with respect to its radius + radii_variance = torch.pow(radii, 2) * ( + t_mu_pow2 / 4 + (5 / 12) * t_delta_pow2 - 4 / 15 * (t_delta_pow4) / den + ) + return rays_dir_mean, rays_dir_variance, radii_variance + + +def conical_frustum_to_gaussian( + ray_bundle: ImplicitronRayBundle, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Approximate a conical frustum following a ray bundle as a Gaussian. + + Args: + ray_bundle: A `RayBundle` or `HeterogeneousRayBundle` object with fields: + origins: A tensor of shape `(..., 3)` + directions: A tensor of shape `(..., 3)` + lengths: A tensor of shape `(..., num_points_per_ray)` + bins: A tensor of shape `(..., num_points_per_ray + 1)` + containing the bins at which the rays are sampled. . + pixel_radii_2d: A tensor of shape `(..., 1)` + base radii of the conical frustums. + + Returns: + means: A tensor of shape `(..., num_points_per_ray - 1, 3)` + representing the means of the Gaussians + approximating the conical frustums. + diag_covariances: A tensor of shape `(...,num_points_per_ray -1, 3)` + representing the diagonal covariance matrices of our Gaussians. + """ + + if ray_bundle.pixel_radii_2d is None or ray_bundle.bins is None: + raise ValueError( + "RayBundle pixel_radii_2d or bins have not been provided." + " Look at pytorch3d.renderer.implicit.renderer.ray_sampler::" + "AbstractMaskRaySampler to see how to compute them. Have you forgot to set" + "`cast_ray_bundle_as_cone` to True?" + ) + + ( + rays_dir_mean, + rays_dir_variance, + radii_variance, + ) = approximate_conical_frustum_as_gaussians( + ray_bundle.bins, + ray_bundle.pixel_radii_2d, + ) + means = ray_bundle_variables_to_ray_points( + ray_bundle.origins, ray_bundle.directions, rays_dir_mean + ) + diag_covariances = compute_3d_diagonal_covariance_gaussian( + ray_bundle.directions, rays_dir_variance, radii_variance + ) + return means, diag_covariances diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/lstm_renderer.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/lstm_renderer.py new file mode 100644 index 0000000000000000000000000000000000000000..0e95dc630283092e2f89d48821a78bb312a49619 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/lstm_renderer.py @@ -0,0 +1,189 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import copy +import logging +from typing import List, Optional, Tuple + +import torch +from pytorch3d.implicitron.models.renderer.base import ImplicitronRayBundle +from pytorch3d.implicitron.tools.config import registry + +from .base import BaseRenderer, EvaluationMode, ImplicitFunctionWrapper, RendererOutput + + +logger = logging.getLogger(__name__) + + +@registry.register +class LSTMRenderer(BaseRenderer, torch.nn.Module): + """ + Implements the learnable LSTM raymarching function from SRN [1]. + This requires there to be one implicit function, and it is expected to be + like SRNImplicitFunction or SRNHyperNetImplicitFunction. + + Settings: + num_raymarch_steps: The number of LSTM raymarching steps. + init_depth: Initializes the bias of the last raymarching LSTM layer so that + the farthest point from the camera reaches a far z-plane that + lies `init_depth` units from the camera plane. + init_depth_noise_std: The standard deviation of the random normal noise + added to the initial depth of each marched ray. + hidden_size: The dimensionality of the LSTM's hidden state. + n_feature_channels: The number of feature channels returned by the + implicit_function evaluated at each raymarching step. + bg_color: If supplied, used as the background color. Otherwise the pixel + generator is used everywhere. This has to have length either 1 + (for a constant value for all output channels) or equal to the number + of output channels (which is `out_features` on the pixel generator, + typically 3.) + verbose: If `True`, logs raymarching debug info. + + References: + [1] Sitzmann, V. and Zollhöfer, M. and Wetzstein, G.. + "Scene representation networks: Continuous 3d-structure-aware + neural scene representations." NeurIPS 2019. + """ + + num_raymarch_steps: int = 10 + init_depth: float = 17.0 + init_depth_noise_std: float = 5e-4 + hidden_size: int = 16 + n_feature_channels: int = 256 + bg_color: Optional[List[float]] = None + verbose: bool = False + + def __post_init__(self): + self._lstm = torch.nn.LSTMCell( + input_size=self.n_feature_channels, + hidden_size=self.hidden_size, + ) + self._lstm.apply(_init_recurrent_weights) + _lstm_forget_gate_init(self._lstm) + self._out_layer = torch.nn.Linear(self.hidden_size, 1) + + one_step = self.init_depth / self.num_raymarch_steps + self._out_layer.bias.data.fill_(one_step) + self._out_layer.weight.data.normal_(mean=0.0, std=1e-3) + + def forward( + self, + ray_bundle: ImplicitronRayBundle, + implicit_functions: List[ImplicitFunctionWrapper], + evaluation_mode: EvaluationMode = EvaluationMode.EVALUATION, + **kwargs, + ) -> RendererOutput: + """ + + Args: + ray_bundle: A `ImplicitronRayBundle` object containing the parametrizations of the + sampled rendering rays. + implicit_functions: A single-element list of ImplicitFunctionWrappers which + defines the implicit function to be used. + evaluation_mode: one of EvaluationMode.TRAINING or + EvaluationMode.EVALUATION which determines the settings used for + rendering, specifically the RayPointRefiner and the density_noise_std. + + Returns: + instance of RendererOutput + """ + if len(implicit_functions) != 1: + raise ValueError("LSTM renderer expects a single implicit function.") + + implicit_function = implicit_functions[0] + + if ray_bundle.lengths.shape[-1] != 1: + raise ValueError( + "LSTM renderer requires a ray-bundle with a single point per ray" + + " which is the initial raymarching point." + ) + + # jitter the initial depths + + ray_bundle_t = copy.copy(ray_bundle) + ray_bundle_t.lengths = ( + ray_bundle.lengths + + torch.randn_like(ray_bundle.lengths) * self.init_depth_noise_std + ) + + states: List[Optional[Tuple[torch.Tensor, torch.Tensor]]] = [None] + signed_distance = torch.zeros_like(ray_bundle_t.lengths) + raymarch_features = None + for t in range(self.num_raymarch_steps + 1): + # move signed_distance along each ray + ray_bundle_t.lengths += signed_distance + + # eval the raymarching function + raymarch_features, _ = implicit_function( + ray_bundle=ray_bundle_t, + raymarch_features=None, + ) + if self.verbose: + msg = ( + f"{t}: mu={float(signed_distance.mean()):1.2e};" + + f" std={float(signed_distance.std()):1.2e};" + + f" mu_d={float(ray_bundle_t.lengths.mean()):1.2e};" + + f" std_d={float(ray_bundle_t.lengths.std()):1.2e};" + ) + logger.info(msg) + if t == self.num_raymarch_steps: + break + + # run the lstm marcher + state_h, state_c = self._lstm( + raymarch_features.view(-1, raymarch_features.shape[-1]), + states[-1], + ) + if state_h.requires_grad: + state_h.register_hook(lambda x: x.clamp(min=-10, max=10)) + # predict the next step size + signed_distance = self._out_layer(state_h).view(ray_bundle_t.lengths.shape) + # log the lstm states + states.append((state_h, state_c)) + + opacity_logits, features = implicit_function( + raymarch_features=raymarch_features, + ray_bundle=ray_bundle_t, + ) + mask = torch.sigmoid(opacity_logits) + depth = ray_bundle_t.lengths * ray_bundle_t.directions.norm( + dim=-1, keepdim=True + ) + + if self.bg_color is not None: + background = features.new_tensor(self.bg_color) + features = torch.lerp(background, features, mask) + + return RendererOutput( + features=features[..., 0, :], + depths=depth, + masks=mask[..., 0, :], + ) + + +def _init_recurrent_weights(self) -> None: + # copied from SRN codebase + for m in self.modules(): + if type(m) in [torch.nn.GRU, torch.nn.LSTM, torch.nn.RNN]: + for name, param in m.named_parameters(): + if "weight_ih" in name: + torch.nn.init.kaiming_normal_(param.data) + elif "weight_hh" in name: + torch.nn.init.orthogonal_(param.data) + elif "bias" in name: + param.data.fill_(0) + + +def _lstm_forget_gate_init(lstm_layer) -> None: + # copied from SRN codebase + for name, parameter in lstm_layer.named_parameters(): + if "bias" not in name: + continue + n = parameter.size(0) + start, end = n // 4, n // 2 + parameter.data[start:end].fill_(1.0) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/multipass_ea.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/multipass_ea.py new file mode 100644 index 0000000000000000000000000000000000000000..905027b0f5739664ef6979b83b23672279f08e9a --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/multipass_ea.py @@ -0,0 +1,187 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from typing import List + +import torch +from pytorch3d.implicitron.models.renderer.base import ImplicitronRayBundle +from pytorch3d.implicitron.tools.config import registry, run_auto_creation + +from .base import BaseRenderer, EvaluationMode, ImplicitFunctionWrapper, RendererOutput +from .ray_point_refiner import RayPointRefiner +from .raymarcher import RaymarcherBase + + +@registry.register +class MultiPassEmissionAbsorptionRenderer( # pyre-ignore: 13 + BaseRenderer, torch.nn.Module +): + """ + Implements the multi-pass rendering function, in particular, + with emission-absorption ray marching used in NeRF [1]. First, it evaluates + opacity-based ray-point weights and then optionally (in case more implicit + functions are given) resamples points using importance sampling and evaluates + new weights. + + During each ray marching pass, features, depth map, and masks + are integrated: Let o_i be the opacity estimated by the implicit function, + and d_i be the offset between points `i` and `i+1` along the respective ray. + Ray marching is performed using the following equations:: + + ray_opacity_n = cap_fn(sum_i=1^n cap_fn(d_i * o_i)), + weight_n = weight_fn(cap_fn(d_i * o_i), 1 - ray_opacity_{n-1}), + + and the final rendered quantities are computed by a dot-product of ray values + with the weights, e.g. `features = sum_n(weight_n * ray_features_n)`. + + By default, for the EA raymarcher from [1] ( + activated with `self.raymarcher_class_type="EmissionAbsorptionRaymarcher"` + ):: + + cap_fn(x) = 1 - exp(-x), + weight_fn(x) = w * x. + + Note that the latter can altered by changing `self.raymarcher_class_type`, + e.g. to "CumsumRaymarcher" which implements the cumulative-sum raymarcher + from NeuralVolumes [2]. + + Settings: + n_pts_per_ray_fine_training: The number of points sampled per ray for the + fine rendering pass during training. + n_pts_per_ray_fine_evaluation: The number of points sampled per ray for the + fine rendering pass during evaluation. + stratified_sampling_coarse_training: Enable/disable stratified sampling in the + refiner during training. Only matters if there are multiple implicit + functions (i.e. in GenericModel if num_passes>1). + stratified_sampling_coarse_evaluation: Enable/disable stratified sampling in + the refiner during evaluation. Only matters if there are multiple implicit + functions (i.e. in GenericModel if num_passes>1). + append_coarse_samples_to_fine: Add the fine ray points to the coarse points + after sampling. + density_noise_std_train: Standard deviation of the noise added to the + opacity field. + return_weights: Enables returning the rendering weights of the EA raymarcher. + Setting to `True` can lead to a prohibitivelly large memory consumption. + blurpool_weights: Use blurpool defined in [3], on the input weights of + each implicit_function except the first (implicit_functions[0]). + sample_pdf_eps: Padding applied to the weights (alpha in equation 18 of [3]). + raymarcher_class_type: The type of self.raymarcher corresponding to + a child of `RaymarcherBase` in the registry. + raymarcher: The raymarcher object used to convert per-point features + and opacities to a feature render. + + References: + [1] Mildenhall, Ben, et al. "Nerf: Representing Scenes as Neural Radiance + Fields for View Synthesis." ECCV 2020. + [2] Lombardi, Stephen, et al. "Neural Volumes: Learning Dynamic Renderable + Volumes from Images." SIGGRAPH 2019. + [3] Jonathan T. Barron, et al. "Mip-NeRF: A Multiscale Representation + for Anti-Aliasing Neural Radiance Fields." ICCV 2021. + + """ + + raymarcher_class_type: str = "EmissionAbsorptionRaymarcher" + raymarcher: RaymarcherBase + + n_pts_per_ray_fine_training: int = 64 + n_pts_per_ray_fine_evaluation: int = 64 + stratified_sampling_coarse_training: bool = True + stratified_sampling_coarse_evaluation: bool = False + append_coarse_samples_to_fine: bool = True + density_noise_std_train: float = 0.0 + return_weights: bool = False + blurpool_weights: bool = False + sample_pdf_eps: float = 1e-5 + + def __post_init__(self): + self._refiners = { + EvaluationMode.TRAINING: RayPointRefiner( + n_pts_per_ray=self.n_pts_per_ray_fine_training, + random_sampling=self.stratified_sampling_coarse_training, + add_input_samples=self.append_coarse_samples_to_fine, + blurpool_weights=self.blurpool_weights, + sample_pdf_eps=self.sample_pdf_eps, + ), + EvaluationMode.EVALUATION: RayPointRefiner( + n_pts_per_ray=self.n_pts_per_ray_fine_evaluation, + random_sampling=self.stratified_sampling_coarse_evaluation, + add_input_samples=self.append_coarse_samples_to_fine, + blurpool_weights=self.blurpool_weights, + sample_pdf_eps=self.sample_pdf_eps, + ), + } + run_auto_creation(self) + + def forward( + self, + ray_bundle: ImplicitronRayBundle, + implicit_functions: List[ImplicitFunctionWrapper], + evaluation_mode: EvaluationMode = EvaluationMode.EVALUATION, + **kwargs, + ) -> RendererOutput: + """ + Args: + ray_bundle: A `ImplicitronRayBundle` object containing the parametrizations of the + sampled rendering rays. + implicit_functions: List of ImplicitFunctionWrappers which + define the implicit functions to be used sequentially in + the raymarching step. The output of raymarching with + implicit_functions[n-1] is refined, and then used as + input for raymarching with implicit_functions[n]. + evaluation_mode: one of EvaluationMode.TRAINING or + EvaluationMode.EVALUATION which determines the settings used for + rendering + + Returns: + instance of RendererOutput + """ + if not implicit_functions: + raise ValueError("EA renderer expects implicit functions") + + return self._run_raymarcher( + ray_bundle, + implicit_functions, + None, + evaluation_mode, + ) + + def _run_raymarcher( + self, ray_bundle, implicit_functions, prev_stage, evaluation_mode + ): + density_noise_std = ( + self.density_noise_std_train + if evaluation_mode == EvaluationMode.TRAINING + else 0.0 + ) + + ray_deltas = ( + None if ray_bundle.bins is None else torch.diff(ray_bundle.bins, dim=-1) + ) + output = self.raymarcher( + *implicit_functions[0](ray_bundle=ray_bundle), + ray_lengths=ray_bundle.lengths, + ray_deltas=ray_deltas, + density_noise_std=density_noise_std, + ) + output.prev_stage = prev_stage + + weights = output.weights + if not self.return_weights: + output.weights = None + + # we may need to make a recursive call + if len(implicit_functions) > 1: + fine_ray_bundle = self._refiners[evaluation_mode](ray_bundle, weights) + output = self._run_raymarcher( + fine_ray_bundle, + implicit_functions[1:], + output, + evaluation_mode, + ) + + return output diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/ray_point_refiner.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/ray_point_refiner.py new file mode 100644 index 0000000000000000000000000000000000000000..90f56c3cc47b010bebab6a1f2b0c3b5f356abea3 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/ray_point_refiner.py @@ -0,0 +1,147 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import copy + +import torch +from pytorch3d.implicitron.models.renderer.base import ImplicitronRayBundle +from pytorch3d.implicitron.tools.config import Configurable, expand_args_fields + +from pytorch3d.renderer.implicit.sample_pdf import sample_pdf + + +@expand_args_fields +# pyre-fixme[13]: Attribute `n_pts_per_ray` is never initialized. +# pyre-fixme[13]: Attribute `random_sampling` is never initialized. +class RayPointRefiner(Configurable, torch.nn.Module): + """ + Implements the importance sampling of points along rays. + The input is a `RayBundle` object with a `ray_weights` tensor + which specifies the probabilities of sampling a point along each ray. + + This raysampler is used for the fine rendering pass of NeRF. + As such, the forward pass accepts the RayBundle output by the + raysampling of the coarse rendering pass. Hence, it does not + take cameras as input. + + Args: + n_pts_per_ray: The number of points to sample along each ray. + random_sampling: If `False`, returns equispaced percentiles of the + distribution defined by the input weights, otherwise performs + sampling from that distribution. + add_input_samples: Concatenates and returns the sampled values + together with the input samples. + blurpool_weights: Use blurpool defined in [1], on the input weights. + sample_pdf_eps: A constant preventing division by zero in case empty bins + are present. + + References: + [1] Jonathan T. Barron, et al. "Mip-NeRF: A Multiscale Representation + for Anti-Aliasing Neural Radiance Fields." ICCV 2021. + """ + + n_pts_per_ray: int + random_sampling: bool + add_input_samples: bool = True + blurpool_weights: bool = False + sample_pdf_eps: float = 1e-5 + + def forward( + self, + input_ray_bundle: ImplicitronRayBundle, + ray_weights: torch.Tensor, + blurpool_weights: bool = False, + sample_pdf_padding: float = 1e-5, + **kwargs, + ) -> ImplicitronRayBundle: + """ + Args: + input_ray_bundle: An instance of `ImplicitronRayBundle` specifying the + source rays for sampling of the probability distribution. + ray_weights: A tensor of shape + `(..., input_ray_bundle.lengths.shape[-1])` with non-negative + elements defining the probability distribution to sample + ray points from. + blurpool_weights: Use blurpool defined in [1], on the input weights. + sample_pdf_padding: A constant preventing division by zero in case empty bins + are present. + + Returns: + ray_bundle: A new `ImplicitronRayBundle` instance containing the input ray + points together with `n_pts_per_ray` additionally sampled + points per ray. For each ray, the lengths are sorted. + + References: + [1] Jonathan T. Barron, et al. "Mip-NeRF: A Multiscale Representation + for Anti-Aliasing Neural Radiance Fields." ICCV 2021. + + """ + + with torch.no_grad(): + if self.blurpool_weights: + ray_weights = apply_blurpool_on_weights(ray_weights) + + n_pts_per_ray = self.n_pts_per_ray + ray_weights = ray_weights.view(-1, ray_weights.shape[-1]) + if input_ray_bundle.bins is None: + z_vals: torch.Tensor = input_ray_bundle.lengths + ray_weights = ray_weights[..., 1:-1] + bins = torch.lerp(z_vals[..., 1:], z_vals[..., :-1], 0.5) + else: + z_vals = input_ray_bundle.bins + n_pts_per_ray += 1 + bins = z_vals + z_samples = sample_pdf( + bins.view(-1, bins.shape[-1]), + ray_weights, + n_pts_per_ray, + det=not self.random_sampling, + eps=self.sample_pdf_eps, + ).view(*z_vals.shape[:-1], n_pts_per_ray) + + if self.add_input_samples: + z_vals = torch.cat((z_vals, z_samples), dim=-1) + else: + z_vals = z_samples + # Resort by depth. + z_vals, _ = torch.sort(z_vals, dim=-1) + ray_bundle = copy.copy(input_ray_bundle) + if input_ray_bundle.bins is None: + ray_bundle.lengths = z_vals + else: + ray_bundle.bins = z_vals + + return ray_bundle + + +def apply_blurpool_on_weights(weights) -> torch.Tensor: + """ + Filter weights with a 2-tap max filters followed by a 2-tap blur filter, + which produces a wide and smooth upper envelope on the weights. + + Args: + weights: Tensor of shape `(..., dim)` + + Returns: + blured_weights: Tensor of shape `(..., dim)` + """ + weights_pad = torch.concatenate( + [ + weights[..., :1], + weights, + weights[..., -1:], + ], + dim=-1, + ) + + weights_max = torch.nn.functional.max_pool1d( + weights_pad.flatten(end_dim=-2), 2, stride=1 + ) + return torch.lerp(weights_max[..., :-1], weights_max[..., 1:], 0.5).reshape_as( + weights + ) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/ray_sampler.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/ray_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..11ee27b045e5a047fd029f7bf361b000119b5358 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/ray_sampler.py @@ -0,0 +1,387 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from typing import Optional, Tuple + +import torch +from pytorch3d.implicitron.tools import camera_utils +from pytorch3d.implicitron.tools.config import registry, ReplaceableBase +from pytorch3d.renderer import NDCMultinomialRaysampler +from pytorch3d.renderer.cameras import CamerasBase +from pytorch3d.renderer.implicit.utils import HeterogeneousRayBundle + +from .base import EvaluationMode, ImplicitronRayBundle, RenderSamplingMode + + +class RaySamplerBase(ReplaceableBase): + """ + Base class for ray samplers. + """ + + def forward( + self, + cameras: CamerasBase, + evaluation_mode: EvaluationMode, + mask: Optional[torch.Tensor] = None, + ) -> ImplicitronRayBundle: + """ + Args: + cameras: A batch of `batch_size` cameras from which the rays are emitted. + evaluation_mode: one of `EvaluationMode.TRAINING` or + `EvaluationMode.EVALUATION` which determines the sampling mode + that is used. + mask: Active for the `RenderSamplingMode.MASK_SAMPLE` sampling mode. + Defines a non-negative mask of shape + `(batch_size, image_height, image_width)` where each per-pixel + value is proportional to the probability of sampling the + corresponding pixel's ray. + + Returns: + ray_bundle: A `ImplicitronRayBundle` object containing the parametrizations of the + sampled rendering rays. + """ + raise NotImplementedError() + + +class AbstractMaskRaySampler(RaySamplerBase, torch.nn.Module): + """ + Samples a fixed number of points along rays which are in turn sampled for + each camera in a batch. + + This class utilizes `NDCMultinomialRaysampler` which allows to either + randomly sample rays from an input foreground saliency mask + (`RenderSamplingMode.MASK_SAMPLE`), or on a rectangular image grid + (`RenderSamplingMode.FULL_GRID`). The sampling mode can be set separately + for training and evaluation by setting `self.sampling_mode_training` + and `self.sampling_mode_training` accordingly. + + The class allows to adjust the sampling points along rays by overwriting the + `AbstractMaskRaySampler._get_min_max_depth_bounds` function which returns + the near/far planes (`min_depth`/`max_depth`) `NDCMultinomialRaysampler`. + + Settings: + image_width: The horizontal size of the image grid. + image_height: The vertical size of the image grid. + sampling_mode_training: The ray sampling mode for training. This should be a str + option from the RenderSamplingMode Enum + sampling_mode_evaluation: Same as above but for evaluation. + n_pts_per_ray_training: The number of points sampled along each ray during training. + n_pts_per_ray_evaluation: The number of points sampled along each ray during evaluation. + n_rays_per_image_sampled_from_mask: The amount of rays to be sampled from the image + grid. Given a batch of image grids, this many is sampled from each. + `n_rays_per_image_sampled_from_mask` and `n_rays_total_training` cannot both be + defined. + n_rays_total_training: (optional) How many rays in total to sample from the entire + batch of provided image grid. The result is as if `n_rays_total_training` + cameras/image grids were sampled with replacement from the cameras / image grids + provided and for every camera one ray was sampled. + `n_rays_per_image_sampled_from_mask` and `n_rays_total_training` cannot both be + defined, to use you have to set `n_rays_per_image` to None. + Used only for EvaluationMode.TRAINING. + stratified_point_sampling_training: if set, performs stratified random sampling + along the ray; otherwise takes ray points at deterministic offsets. + stratified_point_sampling_evaluation: Same as above but for evaluation. + cast_ray_bundle_as_cone: If True, the sampling will generate the bins and radii + attribute of ImplicitronRayBundle. The `bins` contain the z-coordinate + (=depth) of each ray in world units and are of shape + `(batch_size, n_rays_per_image, n_pts_per_ray_training/evaluation + 1)` + while `lengths` is equal to the midpoint of the bins: + (0.5 * (bins[..., 1:] + bins[..., :-1]). + If False, `bins` is None, `radii` is None and `lengths` contains + the z-coordinate (=depth) of each ray in world units and are of shape + `(batch_size, n_rays_per_image, n_pts_per_ray_training/evaluation)` + + Raises: + TypeError: if cast_ray_bundle_as_cone is set to True and n_rays_total_training + is not None will result in an error. HeterogeneousRayBundle is + not supported for conical frustum computation yet. + """ + + image_width: int = 400 + image_height: int = 400 + sampling_mode_training: str = "mask_sample" + sampling_mode_evaluation: str = "full_grid" + n_pts_per_ray_training: int = 64 + n_pts_per_ray_evaluation: int = 64 + n_rays_per_image_sampled_from_mask: Optional[int] = 1024 + n_rays_total_training: Optional[int] = None + # stratified sampling vs taking points at deterministic offsets + stratified_point_sampling_training: bool = True + stratified_point_sampling_evaluation: bool = False + cast_ray_bundle_as_cone: bool = False + + def __post_init__(self): + if (self.n_rays_per_image_sampled_from_mask is not None) and ( + self.n_rays_total_training is not None + ): + raise ValueError( + "Cannot both define n_rays_total_training and " + "n_rays_per_image_sampled_from_mask." + ) + + self._sampling_mode = { + EvaluationMode.TRAINING: RenderSamplingMode(self.sampling_mode_training), + EvaluationMode.EVALUATION: RenderSamplingMode( + self.sampling_mode_evaluation + ), + } + + n_pts_per_ray_training = ( + self.n_pts_per_ray_training + 1 + if self.cast_ray_bundle_as_cone + else self.n_pts_per_ray_training + ) + n_pts_per_ray_evaluation = ( + self.n_pts_per_ray_evaluation + 1 + if self.cast_ray_bundle_as_cone + else self.n_pts_per_ray_evaluation + ) + self._training_raysampler = NDCMultinomialRaysampler( + image_width=self.image_width, + image_height=self.image_height, + n_pts_per_ray=n_pts_per_ray_training, + min_depth=0.0, + max_depth=0.0, + n_rays_per_image=( + self.n_rays_per_image_sampled_from_mask + if self._sampling_mode[EvaluationMode.TRAINING] + == RenderSamplingMode.MASK_SAMPLE + else None + ), + n_rays_total=self.n_rays_total_training, + unit_directions=True, + stratified_sampling=self.stratified_point_sampling_training, + ) + + self._evaluation_raysampler = NDCMultinomialRaysampler( + image_width=self.image_width, + image_height=self.image_height, + n_pts_per_ray=n_pts_per_ray_evaluation, + min_depth=0.0, + max_depth=0.0, + n_rays_per_image=( + self.n_rays_per_image_sampled_from_mask + if self._sampling_mode[EvaluationMode.EVALUATION] + == RenderSamplingMode.MASK_SAMPLE + else None + ), + unit_directions=True, + stratified_sampling=self.stratified_point_sampling_evaluation, + ) + + max_y, min_y = self._training_raysampler.max_y, self._training_raysampler.min_y + max_x, min_x = self._training_raysampler.max_x, self._training_raysampler.min_x + self.pixel_height: float = (max_y - min_y) / (self.image_height - 1) + self.pixel_width: float = (max_x - min_x) / (self.image_width - 1) + + def _get_min_max_depth_bounds(self, cameras: CamerasBase) -> Tuple[float, float]: + raise NotImplementedError() + + def forward( + self, + cameras: CamerasBase, + evaluation_mode: EvaluationMode, + mask: Optional[torch.Tensor] = None, + ) -> ImplicitronRayBundle: + """ + + Args: + cameras: A batch of `batch_size` cameras from which the rays are emitted. + evaluation_mode: one of `EvaluationMode.TRAINING` or + `EvaluationMode.EVALUATION` which determines the sampling mode + that is used. + mask: Active for the `RenderSamplingMode.MASK_SAMPLE` sampling mode. + Defines a non-negative mask of shape + `(batch_size, image_height, image_width)` where each per-pixel + value is proportional to the probability of sampling the + corresponding pixel's ray. + + Returns: + ray_bundle: A `ImplicitronRayBundle` object containing the parametrizations of the + sampled rendering rays. + """ + sample_mask = None + if ( + self._sampling_mode[evaluation_mode] == RenderSamplingMode.MASK_SAMPLE + and mask is not None + ): + sample_mask = torch.nn.functional.interpolate( + mask, + size=[self.image_height, self.image_width], + mode="nearest", + )[:, 0] + + min_depth, max_depth = self._get_min_max_depth_bounds(cameras) + + raysampler = { + EvaluationMode.TRAINING: self._training_raysampler, + EvaluationMode.EVALUATION: self._evaluation_raysampler, + }[evaluation_mode] + + ray_bundle = raysampler( + cameras=cameras, + mask=sample_mask, + min_depth=min_depth, + max_depth=max_depth, + ) + if self.cast_ray_bundle_as_cone and isinstance( + ray_bundle, HeterogeneousRayBundle + ): + # If this error rises it means that raysampler has among + # its arguments `n_ray_totals`. If it is the case + # then you should update the radii computation and lengths + # computation to handle padding and unpadding. + raise TypeError( + "Heterogeneous ray bundle is not supported for conical frustum computation yet" + ) + elif self.cast_ray_bundle_as_cone: + pixel_hw: Tuple[float, float] = (self.pixel_height, self.pixel_width) + pixel_radii_2d = compute_radii(cameras, ray_bundle.xys[..., :2], pixel_hw) + return ImplicitronRayBundle( + directions=ray_bundle.directions, + origins=ray_bundle.origins, + lengths=None, + xys=ray_bundle.xys, + bins=ray_bundle.lengths, + pixel_radii_2d=pixel_radii_2d, + ) + + return ImplicitronRayBundle( + directions=ray_bundle.directions, + origins=ray_bundle.origins, + lengths=ray_bundle.lengths, + xys=ray_bundle.xys, + camera_counts=getattr(ray_bundle, "camera_counts", None), + camera_ids=getattr(ray_bundle, "camera_ids", None), + ) + + +@registry.register +class AdaptiveRaySampler(AbstractMaskRaySampler): + """ + Adaptively samples points on each ray between near and far planes whose + depths are determined based on the distance from the camera center + to a predefined scene center. + + More specifically, + `min_depth = max( + (self.scene_center-camera_center).norm() - self.scene_extent, eps + )` and + `max_depth = (self.scene_center-camera_center).norm() + self.scene_extent`. + + This sampling is ideal for object-centric scenes whose contents are + centered around a known `self.scene_center` and fit into a bounding sphere + with a radius of `self.scene_extent`. + + Args: + scene_center: The xyz coordinates of the center of the scene used + along with `scene_extent` to compute the min and max depth planes + for sampling ray-points. + scene_extent: The radius of the scene bounding box centered at `scene_center`. + """ + + scene_extent: float = 8.0 + scene_center: Tuple[float, float, float] = (0.0, 0.0, 0.0) + + def __post_init__(self): + super().__post_init__() + if self.scene_extent <= 0.0: + raise ValueError("Adaptive raysampler requires self.scene_extent > 0.") + self._scene_center = torch.FloatTensor(self.scene_center) + + def _get_min_max_depth_bounds(self, cameras: CamerasBase) -> Tuple[float, float]: + """ + Returns the adaptively calculated near/far planes. + """ + min_depth, max_depth = camera_utils.get_min_max_depth_bounds( + cameras, self._scene_center, self.scene_extent + ) + return float(min_depth[0]), float(max_depth[0]) + + +@registry.register +class NearFarRaySampler(AbstractMaskRaySampler): + """ + Samples a fixed number of points between fixed near and far z-planes. + Specifically, samples points along each ray with approximately uniform spacing + of z-coordinates between the minimum depth `self.min_depth` and the maximum depth + `self.max_depth`. This sampling is useful for rendering scenes where the camera is + in a constant distance from the focal point of the scene. + + Args: + min_depth: The minimum depth of a ray-point. + max_depth: The maximum depth of a ray-point. + """ + + min_depth: float = 0.1 + max_depth: float = 8.0 + + def _get_min_max_depth_bounds(self, cameras: CamerasBase) -> Tuple[float, float]: + """ + Returns the stored near/far planes. + """ + return self.min_depth, self.max_depth + + +def compute_radii( + cameras: CamerasBase, + xy_grid: torch.Tensor, + pixel_hw_ndc: Tuple[float, float], +) -> torch.Tensor: + """ + Compute radii of conical frustums in world coordinates. + + Args: + cameras: cameras object representing a batch of cameras. + xy_grid: torch.tensor grid of image xy coords. + pixel_hw_ndc: pixel height and width in NDC + + Returns: + radii: A tensor of shape `(..., 1)` radii of a cone. + """ + batch_size = xy_grid.shape[0] + spatial_size = xy_grid.shape[1:-1] + n_rays_per_image = spatial_size.numel() + + xy = xy_grid.view(batch_size, n_rays_per_image, 2) + + # [batch_size, 3 * n_rays_per_image, 2] + xy = torch.cat( + [ + xy, + # Will allow to find the norm on the x axis + xy + torch.tensor([pixel_hw_ndc[1], 0], device=xy.device), + # Will allow to find the norm on the y axis + xy + torch.tensor([0, pixel_hw_ndc[0]], device=xy.device), + ], + dim=1, + ) + # [batch_size, 3 * n_rays_per_image, 3] + xyz = torch.cat( + ( + xy, + xy.new_ones(batch_size, 3 * n_rays_per_image, 1), + ), + dim=-1, + ) + + # unproject the points + unprojected_xyz = cameras.unproject_points(xyz, from_ndc=True) + + plane_world, plane_world_dx, plane_world_dy = torch.split( + unprojected_xyz, n_rays_per_image, dim=1 + ) + + # Distance from each unit-norm direction vector to its neighbors. + dx_norm = torch.linalg.norm(plane_world_dx - plane_world, dim=-1, keepdims=True) + dy_norm = torch.linalg.norm(plane_world_dy - plane_world, dim=-1, keepdims=True) + # Cut the distance in half to obtain the base radius: (dx_norm + dy_norm) * 0.5 + # Scale it by 2/12**0.5 to match the variance of the pixel’s footprint + radii = (dx_norm + dy_norm) / 12**0.5 + + return radii.view(batch_size, *spatial_size, 1) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/ray_tracing.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/ray_tracing.py new file mode 100644 index 0000000000000000000000000000000000000000..9ad46a83e9eee904e61dea6e00306083c0fd4327 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/ray_tracing.py @@ -0,0 +1,592 @@ +# @lint-ignore-every LICENSELINT +# Adapted from https://github.com/lioryariv/idr +# Copyright (c) 2020 Lior Yariv + +# pyre-unsafe + +from typing import Any, Callable, Tuple + +import torch +import torch.nn as nn +from pytorch3d.implicitron.tools.config import Configurable + + +class RayTracing(Configurable, nn.Module): + """ + Finds the intersection points of rays with the implicit surface defined + by a signed distance function (SDF). The algorithm follows the pipeline: + 1. Initialise start and end points on rays by the intersections with + the circumscribing sphere. + 2. Run sphere tracing from both ends. + 3. Divide the untraced segments of non-convergent rays into uniform + intervals and find the one with the sign transition. + 4. Run the secant method to estimate the point of the sign transition. + + Args: + object_bounding_sphere: The radius of the initial sphere circumscribing + the object. + sdf_threshold: Absolute SDF value small enough for the sphere tracer + to consider it a surface. + line_search_step: Length of the backward correction on sphere tracing + iterations. + line_step_iters: Number of backward correction iterations. + sphere_tracing_iters: Maximum number of sphere tracing iterations + (the actual number of iterations may be smaller if all ray + intersections are found). + n_steps: Number of intervals sampled for unconvergent rays. + n_secant_steps: Number of iterations in the secant algorithm. + """ + + object_bounding_sphere: float = 1.0 + sdf_threshold: float = 5.0e-5 + line_search_step: float = 0.5 + line_step_iters: int = 1 + sphere_tracing_iters: int = 10 + n_steps: int = 100 + n_secant_steps: int = 8 + + def forward( + self, + sdf: Callable[[torch.Tensor], torch.Tensor], + cam_loc: torch.Tensor, + object_mask: torch.BoolTensor, + ray_directions: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Args: + sdf: A callable that takes a (N, 3) tensor of points and returns + a tensor of (N,) SDF values. + cam_loc: A tensor of (B, N, 3) ray origins. + object_mask: A (N, 3) tensor of indicators whether a sampled pixel + corresponds to the rendered object or background. + ray_directions: A tensor of (B, N, 3) ray directions. + + Returns: + curr_start_points: A tensor of (B*N, 3) found intersection points + with the implicit surface. + network_object_mask: A tensor of (B*N,) indicators denoting whether + intersections were found. + acc_start_dis: A tensor of (B*N,) distances from the ray origins + to intersrection points. + """ + batch_size, num_pixels, _ = ray_directions.shape + device = cam_loc.device + + sphere_intersections, mask_intersect = _get_sphere_intersection( + cam_loc, ray_directions, r=self.object_bounding_sphere + ) + + ( + curr_start_points, + unfinished_mask_start, + acc_start_dis, + acc_end_dis, + min_dis, + max_dis, + ) = self.sphere_tracing( + batch_size, + num_pixels, + sdf, + cam_loc, + ray_directions, + mask_intersect, + sphere_intersections, + ) + + network_object_mask = acc_start_dis < acc_end_dis + + # The non convergent rays should be handled by the sampler + sampler_mask = unfinished_mask_start + sampler_net_obj_mask = torch.zeros_like( + sampler_mask, dtype=torch.bool, device=device + ) + if sampler_mask.sum() > 0: + sampler_min_max = torch.zeros((batch_size, num_pixels, 2), device=device) + sampler_min_max.reshape(-1, 2)[sampler_mask, 0] = acc_start_dis[ + sampler_mask + ] + sampler_min_max.reshape(-1, 2)[sampler_mask, 1] = acc_end_dis[sampler_mask] + + sampler_pts, sampler_net_obj_mask, sampler_dists = self.ray_sampler( + sdf, cam_loc, object_mask, ray_directions, sampler_min_max, sampler_mask + ) + + curr_start_points[sampler_mask] = sampler_pts[sampler_mask] + acc_start_dis[sampler_mask] = sampler_dists[sampler_mask] + network_object_mask[sampler_mask] = sampler_net_obj_mask[sampler_mask] + + if not self.training: + return curr_start_points, network_object_mask, acc_start_dis + + # in case we are training, we are updating curr_start_points and acc_start_dis for + + ray_directions = ray_directions.reshape(-1, 3) + mask_intersect = mask_intersect.reshape(-1) + # pyre-fixme[9]: object_mask has type `BoolTensor`; used as `Tensor`. + object_mask = object_mask.reshape(-1) + + in_mask = ~network_object_mask & object_mask & ~sampler_mask + out_mask = ~object_mask & ~sampler_mask + + mask_left_out = (in_mask | out_mask) & ~mask_intersect + if ( + mask_left_out.sum() > 0 + ): # project the origin to the not intersect points on the sphere + cam_left_out = cam_loc.reshape(-1, 3)[mask_left_out] + rays_left_out = ray_directions[mask_left_out] + acc_start_dis[mask_left_out] = -torch.bmm( + rays_left_out.view(-1, 1, 3), cam_left_out.view(-1, 3, 1) + ).squeeze() + curr_start_points[mask_left_out] = ( + cam_left_out + acc_start_dis[mask_left_out].unsqueeze(1) * rays_left_out + ) + + mask = (in_mask | out_mask) & mask_intersect + + if mask.sum() > 0: + min_dis[network_object_mask & out_mask] = acc_start_dis[ + network_object_mask & out_mask + ] + + min_mask_points, min_mask_dist = self.minimal_sdf_points( + sdf, cam_loc, ray_directions, mask, min_dis, max_dis + ) + + curr_start_points[mask] = min_mask_points + acc_start_dis[mask] = min_mask_dist + + return curr_start_points, network_object_mask, acc_start_dis + + def sphere_tracing( + self, + batch_size: int, + num_pixels: int, + sdf: Callable[[torch.Tensor], torch.Tensor], + cam_loc: torch.Tensor, + ray_directions: torch.Tensor, + mask_intersect: torch.Tensor, + sphere_intersections: torch.Tensor, + ) -> Tuple[Any, Any, Any, Any, Any, Any]: + """ + Run sphere tracing algorithm for max iterations + from both sides of unit sphere intersection + + Args: + batch_size: + num_pixels: + sdf: + cam_loc: + ray_directions: + mask_intersect: + sphere_intersections: + + Returns: + curr_start_points: + unfinished_mask_start: + acc_start_dis: + acc_end_dis: + min_dis: + max_dis: + """ + + device = cam_loc.device + sphere_intersections_points = ( + cam_loc[..., None, :] + + sphere_intersections[..., None] * ray_directions[..., None, :] + ) + unfinished_mask_start = mask_intersect.reshape(-1).clone() + unfinished_mask_end = mask_intersect.reshape(-1).clone() + + # Initialize start current points + curr_start_points = torch.zeros(batch_size * num_pixels, 3, device=device) + curr_start_points[unfinished_mask_start] = sphere_intersections_points[ + :, :, 0, : + ].reshape(-1, 3)[unfinished_mask_start] + acc_start_dis = torch.zeros(batch_size * num_pixels, device=device) + acc_start_dis[unfinished_mask_start] = sphere_intersections.reshape(-1, 2)[ + unfinished_mask_start, 0 + ] + + # Initialize end current points + curr_end_points = torch.zeros(batch_size * num_pixels, 3, device=device) + curr_end_points[unfinished_mask_end] = sphere_intersections_points[ + :, :, 1, : + ].reshape(-1, 3)[unfinished_mask_end] + acc_end_dis = torch.zeros(batch_size * num_pixels, device=device) + acc_end_dis[unfinished_mask_end] = sphere_intersections.reshape(-1, 2)[ + unfinished_mask_end, 1 + ] + + # Initialise min and max depth + min_dis = acc_start_dis.clone() + max_dis = acc_end_dis.clone() + + # Iterate on the rays (from both sides) till finding a surface + iters = 0 + + # TODO: sdf should also pass info about batches + + next_sdf_start = torch.zeros_like(acc_start_dis) + next_sdf_start[unfinished_mask_start] = sdf( + curr_start_points[unfinished_mask_start] + ) + + next_sdf_end = torch.zeros_like(acc_end_dis) + next_sdf_end[unfinished_mask_end] = sdf(curr_end_points[unfinished_mask_end]) + + while True: + # Update sdf + curr_sdf_start = torch.zeros_like(acc_start_dis) + curr_sdf_start[unfinished_mask_start] = next_sdf_start[ + unfinished_mask_start + ] + curr_sdf_start[curr_sdf_start <= self.sdf_threshold] = 0 + + curr_sdf_end = torch.zeros_like(acc_end_dis) + curr_sdf_end[unfinished_mask_end] = next_sdf_end[unfinished_mask_end] + curr_sdf_end[curr_sdf_end <= self.sdf_threshold] = 0 + + # Update masks + unfinished_mask_start = unfinished_mask_start & ( + curr_sdf_start > self.sdf_threshold + ) + unfinished_mask_end = unfinished_mask_end & ( + curr_sdf_end > self.sdf_threshold + ) + + if ( + unfinished_mask_start.sum() == 0 and unfinished_mask_end.sum() == 0 + ) or iters == self.sphere_tracing_iters: + break + iters += 1 + + # Make step + # Update distance + acc_start_dis = acc_start_dis + curr_sdf_start + acc_end_dis = acc_end_dis - curr_sdf_end + + # Update points + curr_start_points = ( + cam_loc + + acc_start_dis.reshape(batch_size, num_pixels, 1) * ray_directions + ).reshape(-1, 3) + curr_end_points = ( + cam_loc + + acc_end_dis.reshape(batch_size, num_pixels, 1) * ray_directions + ).reshape(-1, 3) + + # Fix points which wrongly crossed the surface + next_sdf_start = torch.zeros_like(acc_start_dis) + next_sdf_start[unfinished_mask_start] = sdf( + curr_start_points[unfinished_mask_start] + ) + + next_sdf_end = torch.zeros_like(acc_end_dis) + next_sdf_end[unfinished_mask_end] = sdf( + curr_end_points[unfinished_mask_end] + ) + + not_projected_start = next_sdf_start < 0 + not_projected_end = next_sdf_end < 0 + not_proj_iters = 0 + while ( + not_projected_start.sum() > 0 or not_projected_end.sum() > 0 + ) and not_proj_iters < self.line_step_iters: + # Step backwards + acc_start_dis[not_projected_start] -= ( + (1 - self.line_search_step) / (2**not_proj_iters) + ) * curr_sdf_start[not_projected_start] + curr_start_points[not_projected_start] = ( + cam_loc + + acc_start_dis.reshape(batch_size, num_pixels, 1) * ray_directions + ).reshape(-1, 3)[not_projected_start] + + acc_end_dis[not_projected_end] += ( + (1 - self.line_search_step) / (2**not_proj_iters) + ) * curr_sdf_end[not_projected_end] + curr_end_points[not_projected_end] = ( + cam_loc + + acc_end_dis.reshape(batch_size, num_pixels, 1) * ray_directions + ).reshape(-1, 3)[not_projected_end] + + # Calc sdf + next_sdf_start[not_projected_start] = sdf( + curr_start_points[not_projected_start] + ) + next_sdf_end[not_projected_end] = sdf( + curr_end_points[not_projected_end] + ) + + # Update mask + not_projected_start = next_sdf_start < 0 + not_projected_end = next_sdf_end < 0 + not_proj_iters += 1 + + unfinished_mask_start = unfinished_mask_start & ( + acc_start_dis < acc_end_dis + ) + unfinished_mask_end = unfinished_mask_end & (acc_start_dis < acc_end_dis) + + return ( + curr_start_points, + unfinished_mask_start, + acc_start_dis, + acc_end_dis, + min_dis, + max_dis, + ) + + def ray_sampler( + self, + sdf: Callable[[torch.Tensor], torch.Tensor], + cam_loc: torch.Tensor, + object_mask: torch.Tensor, + ray_directions: torch.Tensor, + sampler_min_max: torch.Tensor, + sampler_mask: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Sample the ray in a given range and run secant on rays which have sign transition. + + Args: + sdf: + cam_loc: + object_mask: + ray_directions: + sampler_min_max: + sampler_mask: + + Returns: + + """ + + batch_size, num_pixels, _ = ray_directions.shape + device = cam_loc.device + n_total_pxl = batch_size * num_pixels + sampler_pts = torch.zeros(n_total_pxl, 3, device=device) + sampler_dists = torch.zeros(n_total_pxl, device=device) + + intervals_dist = torch.linspace(0, 1, steps=self.n_steps, device=device).view( + 1, 1, -1 + ) + + pts_intervals = sampler_min_max[:, :, 0].unsqueeze(-1) + intervals_dist * ( + sampler_min_max[:, :, 1] - sampler_min_max[:, :, 0] + ).unsqueeze(-1) + points = ( + cam_loc[..., None, :] + + pts_intervals[..., None] * ray_directions[..., None, :] + ) + + # Get the non convergent rays + mask_intersect_idx = torch.nonzero(sampler_mask).flatten() + points = points.reshape((-1, self.n_steps, 3))[sampler_mask, :, :] + pts_intervals = pts_intervals.reshape((-1, self.n_steps))[sampler_mask] + + sdf_val_all = [] + for pnts in torch.split(points.reshape(-1, 3), 100000, dim=0): + sdf_val_all.append(sdf(pnts)) + sdf_val = torch.cat(sdf_val_all).reshape(-1, self.n_steps) + + tmp = torch.sign(sdf_val) * torch.arange( + self.n_steps, 0, -1, device=device, dtype=torch.float32 + ).reshape(1, self.n_steps) + # Force argmin to return the first min value + sampler_pts_ind = torch.argmin(tmp, -1) + sampler_pts[mask_intersect_idx] = points[ + torch.arange(points.shape[0]), sampler_pts_ind, : + ] + sampler_dists[mask_intersect_idx] = pts_intervals[ + torch.arange(pts_intervals.shape[0]), sampler_pts_ind + ] + + true_surface_pts = object_mask.reshape(-1)[sampler_mask] + net_surface_pts = sdf_val[torch.arange(sdf_val.shape[0]), sampler_pts_ind] < 0 + + # take points with minimal SDF value for P_out pixels + p_out_mask = ~(true_surface_pts & net_surface_pts) + n_p_out = p_out_mask.sum() + if n_p_out > 0: + out_pts_idx = torch.argmin(sdf_val[p_out_mask, :], -1) + sampler_pts[mask_intersect_idx[p_out_mask]] = points[p_out_mask, :, :][ + # pyre-fixme[6]: For 1st param expected `Union[bool, float, int]` + # but got `Tensor`. + torch.arange(n_p_out), + out_pts_idx, + :, + ] + sampler_dists[mask_intersect_idx[p_out_mask]] = pts_intervals[ + p_out_mask, + :, + # pyre-fixme[6]: For 1st param expected `Union[bool, float, int]` but + # got `Tensor`. + ][torch.arange(n_p_out), out_pts_idx] + + # Get Network object mask + sampler_net_obj_mask = sampler_mask.clone() + sampler_net_obj_mask[mask_intersect_idx[~net_surface_pts]] = False + + # Run Secant method + secant_pts = ( + net_surface_pts & true_surface_pts if self.training else net_surface_pts + ) + n_secant_pts = secant_pts.sum() + if n_secant_pts > 0: + # Get secant z predictions + z_high = pts_intervals[ + torch.arange(pts_intervals.shape[0]), sampler_pts_ind + ][secant_pts] + sdf_high = sdf_val[torch.arange(sdf_val.shape[0]), sampler_pts_ind][ + secant_pts + ] + z_low = pts_intervals[secant_pts][ + # pyre-fixme[6]: For 1st param expected `Union[bool, float, int]` + # but got `Tensor`. + torch.arange(n_secant_pts), + sampler_pts_ind[secant_pts] - 1, + ] + sdf_low = sdf_val[secant_pts][ + # pyre-fixme[6]: For 1st param expected `Union[bool, float, int]` + # but got `Tensor`. + torch.arange(n_secant_pts), + sampler_pts_ind[secant_pts] - 1, + ] + cam_loc_secant = cam_loc.reshape(-1, 3)[mask_intersect_idx[secant_pts]] + ray_directions_secant = ray_directions.reshape((-1, 3))[ + mask_intersect_idx[secant_pts] + ] + z_pred_secant = self.secant( + sdf_low, + sdf_high, + z_low, + z_high, + cam_loc_secant, + ray_directions_secant, + # pyre-fixme[6]: For 7th param expected `Module` but got `(Tensor) + # -> Tensor`. + sdf, + ) + + # Get points + sampler_pts[mask_intersect_idx[secant_pts]] = ( + cam_loc_secant + z_pred_secant.unsqueeze(-1) * ray_directions_secant + ) + sampler_dists[mask_intersect_idx[secant_pts]] = z_pred_secant + + return sampler_pts, sampler_net_obj_mask, sampler_dists + + def secant( + self, + sdf_low: torch.Tensor, + sdf_high: torch.Tensor, + z_low: torch.Tensor, + z_high: torch.Tensor, + cam_loc: torch.Tensor, + ray_directions: torch.Tensor, + sdf: nn.Module, + ) -> torch.Tensor: + """ + Runs the secant method for interval [z_low, z_high] for n_secant_steps + """ + + z_pred = -sdf_low * (z_high - z_low) / (sdf_high - sdf_low) + z_low + for _ in range(self.n_secant_steps): + p_mid = cam_loc + z_pred.unsqueeze(-1) * ray_directions + sdf_mid = sdf(p_mid) + ind_low = sdf_mid > 0 + if ind_low.sum() > 0: + z_low[ind_low] = z_pred[ind_low] + sdf_low[ind_low] = sdf_mid[ind_low] + ind_high = sdf_mid < 0 + if ind_high.sum() > 0: + z_high[ind_high] = z_pred[ind_high] + sdf_high[ind_high] = sdf_mid[ind_high] + + z_pred = -sdf_low * (z_high - z_low) / (sdf_high - sdf_low) + z_low + + return z_pred + + def minimal_sdf_points( + self, + sdf: Callable[[torch.Tensor], torch.Tensor], + cam_loc: torch.Tensor, + ray_directions: torch.Tensor, + mask: torch.Tensor, + min_dis: torch.Tensor, + max_dis: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Find points with minimal SDF value on rays for P_out pixels + """ + + n_mask_points = mask.sum() + + n = self.n_steps + steps = torch.empty(n, device=cam_loc.device).uniform_(0.0, 1.0) + mask_max_dis = max_dis[mask].unsqueeze(-1) + mask_min_dis = min_dis[mask].unsqueeze(-1) + steps = ( + # pyre-fixme[6]: For 1st param expected `int` but got `Tensor`. + steps.unsqueeze(0).repeat(n_mask_points, 1) * (mask_max_dis - mask_min_dis) + + mask_min_dis + ) + + mask_points = cam_loc.reshape(-1, 3)[mask] + mask_rays = ray_directions[mask, :] + + mask_points_all = mask_points.unsqueeze(1).repeat(1, n, 1) + steps.unsqueeze( + -1 + ) * mask_rays.unsqueeze(1).repeat(1, n, 1) + points = mask_points_all.reshape(-1, 3) + + mask_sdf_all = [] + for pnts in torch.split(points, 100000, dim=0): + mask_sdf_all.append(sdf(pnts)) + + mask_sdf_all = torch.cat(mask_sdf_all).reshape(-1, n) + min_vals, min_idx = mask_sdf_all.min(-1) + min_mask_points = mask_points_all.reshape(-1, n, 3)[ + # pyre-fixme[6]: For 2nd param expected `Union[bool, float, int]` but + # got `Tensor`. + torch.arange(0, n_mask_points), + min_idx, + ] + # pyre-fixme[6]: For 2nd param expected `Union[bool, float, int]` but got + # `Tensor`. + min_mask_dist = steps.reshape(-1, n)[torch.arange(0, n_mask_points), min_idx] + + return min_mask_points, min_mask_dist + + +# TODO: support variable origins +def _get_sphere_intersection( + cam_loc: torch.Tensor, ray_directions: torch.Tensor, r: float = 1.0 +) -> Tuple[torch.Tensor, torch.Tensor]: + # Input: n_images x 3 ; n_images x n_rays x 3 + # Output: n_images * n_rays x 2 (close and far) ; n_images * n_rays + + n_imgs, n_pix, _ = ray_directions.shape + device = cam_loc.device + + # cam_loc = cam_loc.unsqueeze(-1) + # ray_cam_dot = torch.bmm(ray_directions, cam_loc).squeeze() + ray_cam_dot = (ray_directions * cam_loc).sum(-1) # n_images x n_rays + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and `int`. + under_sqrt = ray_cam_dot**2 - (cam_loc.norm(2, dim=-1) ** 2 - r**2) + + under_sqrt = under_sqrt.reshape(-1) + mask_intersect = under_sqrt > 0 + + sphere_intersections = torch.zeros(n_imgs * n_pix, 2, device=device) + sphere_intersections[mask_intersect] = torch.sqrt( + under_sqrt[mask_intersect] + ).unsqueeze(-1) * torch.tensor([-1.0, 1.0], device=device) + sphere_intersections[mask_intersect] -= ray_cam_dot.reshape(-1)[ + mask_intersect + ].unsqueeze(-1) + + sphere_intersections = sphere_intersections.reshape(n_imgs, n_pix, 2) + sphere_intersections = sphere_intersections.clamp_min(0.0) + mask_intersect = mask_intersect.reshape(n_imgs, n_pix) + + return sphere_intersections, mask_intersect diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/raymarcher.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/raymarcher.py new file mode 100644 index 0000000000000000000000000000000000000000..5544590cf807243b7741f9452ed9169d0aed9063 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/raymarcher.py @@ -0,0 +1,240 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from typing import Any, Callable, Dict, Optional, Tuple + +import torch +from pytorch3d.implicitron.models.renderer.base import RendererOutput +from pytorch3d.implicitron.tools.config import registry, ReplaceableBase +from pytorch3d.renderer.implicit.raymarching import _check_raymarcher_inputs + + +_TTensor = torch.Tensor + + +class RaymarcherBase(ReplaceableBase): + """ + Defines a base class for raymarchers. Specifically, a raymarcher is responsible + for taking a set of features and density descriptors along rendering rays + and marching along them in order to generate a feature render. + """ + + def forward( + self, + rays_densities: torch.Tensor, + rays_features: torch.Tensor, + aux: Dict[str, Any], + ) -> RendererOutput: + """ + Args: + rays_densities: Per-ray density values represented with a tensor + of shape `(..., n_points_per_ray, 1)`. + rays_features: Per-ray feature values represented with a tensor + of shape `(..., n_points_per_ray, feature_dim)`. + aux: a dictionary with extra information. + """ + raise NotImplementedError() + + +class AccumulativeRaymarcherBase(RaymarcherBase, torch.nn.Module): + """ + This generalizes the `pytorch3d.renderer.EmissionAbsorptionRaymarcher` + and NeuralVolumes' cumsum ray marcher. It additionally returns + the rendering weights that can be used in the NVS pipeline to carry out + the importance ray-sampling in the refining pass. + Different from `pytorch3d.renderer.EmissionAbsorptionRaymarcher`, it takes raw + (non-exponentiated) densities. + + Args: + surface_thickness: The thickness of the raymarched surface. + bg_color: The background color. A tuple of either 1 element or of D elements, + where D matches the feature dimensionality; it is broadcast when necessary. + replicate_last_interval: If True, the ray length assigned to the last interval + for the opacity delta calculation is copied from the penultimate interval. + background_opacity: The length over which the last raw opacity value + (i.e. before exponentiation) is considered to apply, for the delta + calculation. Ignored if replicate_last_interval=True. + density_relu: If `True`, passes the input density through ReLU before + raymarching. + blend_output: If `True`, alpha-blends the output renders with the + background color using the rendered opacity mask. + + capping_function: The capping function of the raymarcher. + Options: + - "exponential" (`cap_fn(x) = 1 - exp(-x)`) + - "cap1" (`cap_fn(x) = min(x, 1)`) + Set to "exponential" for the standard Emission Absorption raymarching. + weight_function: The weighting function of the raymarcher. + Options: + - "product" (`weight_fn(w, x) = w * x`) + - "minimum" (`weight_fn(w, x) = min(w, x)`) + Set to "product" for the standard Emission Absorption raymarching. + """ + + surface_thickness: int = 1 + bg_color: Tuple[float, ...] = (0.0,) + replicate_last_interval: bool = False + background_opacity: float = 0.0 + density_relu: bool = True + blend_output: bool = False + + @property + def capping_function_type(self) -> str: + raise NotImplementedError() + + @property + def weight_function_type(self) -> str: + raise NotImplementedError() + + def __post_init__(self): + """ + Args: + surface_thickness: Denotes the overlap between the absorption + function and the density function. + """ + bg_color = torch.tensor(self.bg_color) + if bg_color.ndim != 1: + raise ValueError(f"bg_color (shape {bg_color.shape}) should be a 1D tensor") + + self.register_buffer("_bg_color", bg_color, persistent=False) + + self._capping_function: Callable[[_TTensor], _TTensor] = { + "exponential": lambda x: 1.0 - torch.exp(-x), + "cap1": lambda x: x.clamp(max=1.0), + }[self.capping_function_type] + + self._weight_function: Callable[[_TTensor, _TTensor], _TTensor] = { + "product": lambda curr, acc: curr * acc, + "minimum": lambda curr, acc: torch.minimum(curr, acc), + }[self.weight_function_type] + + # pyre-fixme[14]: `forward` overrides method defined in `RaymarcherBase` + # inconsistently. + def forward( + self, + rays_densities: torch.Tensor, + rays_features: torch.Tensor, + aux: Dict[str, Any], + ray_lengths: torch.Tensor, + ray_deltas: Optional[torch.Tensor] = None, + density_noise_std: float = 0.0, + **kwargs, + ) -> RendererOutput: + """ + Args: + rays_densities: Per-ray density values represented with a tensor + of shape `(..., n_points_per_ray, 1)`. + rays_features: Per-ray feature values represented with a tensor + of shape `(..., n_points_per_ray, feature_dim)`. + aux: a dictionary with extra information. + ray_lengths: Per-ray depth values represented with a tensor + of shape `(..., n_points_per_ray, feature_dim)`. + ray_deltas: Optional differences between consecutive elements along the ray bundle + represented with a tensor of shape `(..., n_points_per_ray)`. If None, + these differences are computed from ray_lengths. + density_noise_std: the magnitude of the noise added to densities. + + Returns: + features: A tensor of shape `(..., feature_dim)` containing + the rendered features for each ray. + depth: A tensor of shape `(..., 1)` containing estimated depth. + opacities: A tensor of shape `(..., 1)` containing rendered opacities. + weights: A tensor of shape `(..., n_points_per_ray)` containing + the ray-specific non-negative opacity weights. In general, they + don't sum to 1 but do not overcome it, i.e. + `(weights.sum(dim=-1) <= 1.0).all()` holds. + """ + _check_raymarcher_inputs( + rays_densities, + rays_features, + ray_lengths, + z_can_be_none=True, + features_can_be_none=False, + density_1d=True, + ) + + if ray_deltas is None: + ray_lengths_diffs = torch.diff(ray_lengths, dim=-1) + if self.replicate_last_interval: + last_interval = ray_lengths_diffs[..., -1:] + else: + last_interval = torch.full_like( + ray_lengths[..., :1], self.background_opacity + ) + deltas = torch.cat((ray_lengths_diffs, last_interval), dim=-1) + else: + deltas = ray_deltas + + rays_densities = rays_densities[..., 0] + + if density_noise_std > 0.0: + noise: _TTensor = torch.randn_like(rays_densities).mul(density_noise_std) + rays_densities = rays_densities + noise + if self.density_relu: + rays_densities = torch.relu(rays_densities) + + weighted_densities = deltas * rays_densities + capped_densities = self._capping_function(weighted_densities) + + rays_opacities = self._capping_function( + torch.cumsum(weighted_densities, dim=-1) + ) + opacities = rays_opacities[..., -1:] + absorption_shifted = (-rays_opacities + 1.0).roll( + self.surface_thickness, dims=-1 + ) + absorption_shifted[..., : self.surface_thickness] = 1.0 + + weights = self._weight_function(capped_densities, absorption_shifted) + features = (weights[..., None] * rays_features).sum(dim=-2) + depth = (weights * ray_lengths)[..., None].sum(dim=-2) + + alpha = opacities if self.blend_output else 1 + if self._bg_color.shape[-1] not in [1, features.shape[-1]]: + raise ValueError("Wrong number of background color channels.") + features = alpha * features + (1 - opacities) * self._bg_color + + return RendererOutput( + features=features, + depths=depth, + masks=opacities, + weights=weights, + aux=aux, + ) + + +@registry.register +class EmissionAbsorptionRaymarcher(AccumulativeRaymarcherBase): + """ + Implements the EmissionAbsorption raymarcher. + """ + + background_opacity: float = 1e10 + + @property + def capping_function_type(self) -> str: + return "exponential" + + @property + def weight_function_type(self) -> str: + return "product" + + +@registry.register +class CumsumRaymarcher(AccumulativeRaymarcherBase): + """ + Implements the NeuralVolumes' cumulative-sum raymarcher. + """ + + @property + def capping_function_type(self) -> str: + return "cap1" + + @property + def weight_function_type(self) -> str: + return "minimum" diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/rgb_net.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/rgb_net.py new file mode 100644 index 0000000000000000000000000000000000000000..de25389680878ac00d9970d6f6dfb6eed1874ca8 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/rgb_net.py @@ -0,0 +1,140 @@ +# @lint-ignore-every LICENSELINT +# Adapted from RenderingNetwork from IDR +# https://github.com/lioryariv/idr/ +# Copyright (c) 2020 Lior Yariv + +# pyre-unsafe + +import logging +from typing import List, Tuple + +import torch +from pytorch3d.implicitron.models.renderer.base import ImplicitronRayBundle +from pytorch3d.implicitron.tools.config import enable_get_default_args +from pytorch3d.renderer.implicit import HarmonicEmbedding + +from torch import nn + + +logger = logging.getLogger(__name__) + + +class RayNormalColoringNetwork(torch.nn.Module): + """ + Members: + d_in and feature_vector_size: Sum of these is the input + dimension. These must add up to the sum of + - 3 [for the points] + - 3 unless mode=no_normal [for the normals] + - 3 unless mode=no_view_dir [for view directions] + - the feature size, [number of channels in feature_vectors] + + d_out: dimension of output. + mode: One of "idr", "no_view_dir" or "no_normal" to allow omitting + part of the network input. + dims: list of hidden layer sizes. + weight_norm: whether to apply weight normalization to each layer. + n_harmonic_functions_dir: + If >0, use a harmonic embedding with this number of + harmonic functions for the view direction. Otherwise view directions + are fed without embedding, unless mode is `no_view_dir`. + pooled_feature_dim: If a pooling function is in use (provided as + pooling_fn to forward()) this must be its number of features. + Otherwise this must be set to 0. (If used from GenericModel, + this will be set automatically.) + """ + + def __init__( + self, + feature_vector_size: int = 3, + mode: str = "idr", + d_in: int = 9, + d_out: int = 3, + dims: Tuple[int, ...] = (512, 512, 512, 512), + weight_norm: bool = True, + n_harmonic_functions_dir: int = 0, + pooled_feature_dim: int = 0, + ) -> None: + super().__init__() + + self.mode = mode + self.output_dimensions = d_out + dims_full: List[int] = [d_in + feature_vector_size] + list(dims) + [d_out] + + self.embedview_fn = None + if n_harmonic_functions_dir > 0: + self.embedview_fn = HarmonicEmbedding( + n_harmonic_functions_dir, append_input=True + ) + dims_full[0] += self.embedview_fn.get_output_dim() - 3 + + if pooled_feature_dim > 0: + logger.info("Pooled features in rendering network.") + dims_full[0] += pooled_feature_dim + + self.num_layers = len(dims_full) + + layers = [] + for layer_idx in range(self.num_layers - 1): + out_dim = dims_full[layer_idx + 1] + lin = nn.Linear(dims_full[layer_idx], out_dim) + + if weight_norm: + lin = nn.utils.weight_norm(lin) + + layers.append(lin) + self.linear_layers = torch.nn.ModuleList(layers) + + self.relu = nn.ReLU() + self.tanh = nn.Tanh() + + def forward( + self, + feature_vectors: torch.Tensor, + points, + normals, + ray_bundle: ImplicitronRayBundle, + masks=None, + pooling_fn=None, + ): + if masks is not None and not masks.any(): + return torch.zeros_like(normals) + + view_dirs = ray_bundle.directions + if masks is not None: + # in case of IDR, other outputs are passed here after applying the mask + view_dirs = view_dirs.reshape(view_dirs.shape[0], -1, 3)[ + :, masks.reshape(-1) + ] + + if self.embedview_fn is not None: + view_dirs = self.embedview_fn(view_dirs) + + if self.mode == "idr": + rendering_input = torch.cat( + [points, view_dirs, normals, feature_vectors], dim=-1 + ) + elif self.mode == "no_view_dir": + rendering_input = torch.cat([points, normals, feature_vectors], dim=-1) + elif self.mode == "no_normal": + rendering_input = torch.cat([points, view_dirs, feature_vectors], dim=-1) + else: + raise ValueError(f"Unsupported rendering mode: {self.mode}") + + if pooling_fn is not None: + featspool = pooling_fn(points[None])[0] + rendering_input = torch.cat((rendering_input, featspool), dim=-1) + + x = rendering_input + + for layer_idx in range(self.num_layers - 1): + x = self.linear_layers[layer_idx](x) + + if layer_idx < self.num_layers - 2: + x = self.relu(x) + + x = self.tanh(x) + return x + + +enable_get_default_args(RayNormalColoringNetwork) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/sdf_renderer.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/sdf_renderer.py new file mode 100644 index 0000000000000000000000000000000000000000..67ee750996ba52b7a227cf8c7889841bf75b81f5 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/renderer/sdf_renderer.py @@ -0,0 +1,276 @@ +# @lint-ignore-every LICENSELINT +# Adapted from https://github.com/lioryariv/idr/blob/main/code/model/ +# implicit_differentiable_renderer.py +# Copyright (c) 2020 Lior Yariv + +# pyre-unsafe +import functools +from typing import List, Optional, Tuple + +import torch +from omegaconf import DictConfig +from pytorch3d.common.compat import prod +from pytorch3d.implicitron.models.renderer.base import ImplicitronRayBundle +from pytorch3d.implicitron.tools.config import ( + get_default_args_field, + registry, + run_auto_creation, +) +from pytorch3d.implicitron.tools.utils import evaluating + +from .base import BaseRenderer, EvaluationMode, ImplicitFunctionWrapper, RendererOutput +from .ray_tracing import RayTracing +from .rgb_net import RayNormalColoringNetwork + + +@registry.register +class SignedDistanceFunctionRenderer(BaseRenderer, torch.nn.Module): # pyre-ignore[13] + render_features_dimensions: int = 3 + object_bounding_sphere: float = 1.0 + ray_tracer: RayTracing + ray_normal_coloring_network_args: DictConfig = get_default_args_field( + RayNormalColoringNetwork + ) + bg_color: Tuple[float, ...] = (0.0,) + soft_mask_alpha: float = 50.0 + + def __post_init__( + self, + ): + render_features_dimensions = self.render_features_dimensions + if len(self.bg_color) not in [1, render_features_dimensions]: + raise ValueError( + f"Background color should have {render_features_dimensions} entries." + ) + + run_auto_creation(self) + + self.ray_normal_coloring_network_args["feature_vector_size"] = ( + render_features_dimensions + ) + self._rgb_network = RayNormalColoringNetwork( + **self.ray_normal_coloring_network_args + ) + + self.register_buffer("_bg_color", torch.tensor(self.bg_color), persistent=False) + + @classmethod + def ray_tracer_tweak_args(cls, type, args: DictConfig) -> None: + del args["object_bounding_sphere"] + + def create_ray_tracer(self) -> None: + self.ray_tracer = RayTracing( + **self.ray_tracer_args, + object_bounding_sphere=self.object_bounding_sphere, + ) + + def requires_object_mask(self) -> bool: + return True + + def forward( + self, + ray_bundle: ImplicitronRayBundle, + implicit_functions: List[ImplicitFunctionWrapper], + evaluation_mode: EvaluationMode = EvaluationMode.EVALUATION, + object_mask: Optional[torch.Tensor] = None, + **kwargs, + ) -> RendererOutput: + """ + Args: + ray_bundle: A `ImplicitronRayBundle` object containing the parametrizations of the + sampled rendering rays. + implicit_functions: single element list of ImplicitFunctionWrappers which + defines the implicit function to be used. + evaluation_mode: one of EvaluationMode.TRAINING or + EvaluationMode.EVALUATION which determines the settings used for + rendering. + kwargs: + object_mask: BoolTensor, denoting the silhouette of the object. + This is a required keyword argument for SignedDistanceFunctionRenderer + + Returns: + instance of RendererOutput + """ + if len(implicit_functions) != 1: + raise ValueError( + "SignedDistanceFunctionRenderer supports only single pass." + ) + + if object_mask is None: + raise ValueError("Expected object_mask to be provided in the kwargs") + object_mask = object_mask.bool() + + implicit_function = implicit_functions[0] + implicit_function_gradient = functools.partial(_gradient, implicit_function) + + # object_mask: silhouette of the object + batch_size, *spatial_size, _ = ray_bundle.lengths.shape + num_pixels = prod(spatial_size) + + cam_loc = ray_bundle.origins.reshape(batch_size, -1, 3) + ray_dirs = ray_bundle.directions.reshape(batch_size, -1, 3) + object_mask = object_mask.reshape(batch_size, -1) + + with torch.no_grad(), evaluating(implicit_function): + points, network_object_mask, dists = self.ray_tracer( + sdf=lambda x: implicit_function(rays_points_world=x)[ + :, 0 + ], # TODO: get rid of this wrapper + cam_loc=cam_loc, + object_mask=object_mask, + ray_directions=ray_dirs, + ) + + # TODO: below, cam_loc might as well be different + depth = dists.reshape(batch_size, num_pixels, 1) + points = (cam_loc + depth * ray_dirs).reshape(-1, 3) + + sdf_output = implicit_function(rays_points_world=points)[:, 0:1] + # NOTE most of the intermediate variables are flattened for + # no apparent reason (here and in the ray tracer) + ray_dirs = ray_dirs.reshape(-1, 3) + object_mask = object_mask.reshape(-1) + + # TODO: move it to loss computation + if evaluation_mode == EvaluationMode.TRAINING: + surface_mask = network_object_mask & object_mask + surface_points = points[surface_mask] + surface_dists = dists[surface_mask].unsqueeze(-1) + surface_ray_dirs = ray_dirs[surface_mask] + surface_cam_loc = cam_loc.reshape(-1, 3)[surface_mask] + surface_output = sdf_output[surface_mask] + N = surface_points.shape[0] + + # Sample points for the eikonal loss + eik_bounding_box: float = self.object_bounding_sphere + n_eik_points = batch_size * num_pixels // 2 + eikonal_points = torch.empty( + n_eik_points, + 3, + # but got `Union[device, Tensor, Module]`. + device=self._bg_color.device, + ).uniform_(-eik_bounding_box, eik_bounding_box) + eikonal_pixel_points = points.clone() + eikonal_pixel_points = eikonal_pixel_points.detach() + eikonal_points = torch.cat([eikonal_points, eikonal_pixel_points], 0) + + points_all = torch.cat([surface_points, eikonal_points], dim=0) + + output = implicit_function(rays_points_world=surface_points) + surface_sdf_values = output[ + :N, 0:1 + ].detach() # how is it different from sdf_output? + + g = implicit_function_gradient(points_all) + surface_points_grad = g[:N, 0, :].clone().detach() + grad_theta = g[N:, 0, :] + + differentiable_surface_points = _sample_network( + surface_output, + surface_sdf_values, + surface_points_grad, + surface_dists, + surface_cam_loc, + surface_ray_dirs, + ) + + else: + surface_mask = network_object_mask + differentiable_surface_points = points[surface_mask] + grad_theta = None + + empty_render = differentiable_surface_points.shape[0] == 0 + features = implicit_function(rays_points_world=differentiable_surface_points)[ + None, :, 1: + ] + normals_full = features.new_zeros( + batch_size, *spatial_size, 3, requires_grad=empty_render + ) + render_full = ( + features.new_ones( + batch_size, + *spatial_size, + self.render_features_dimensions, + requires_grad=empty_render, + ) + * self._bg_color + ) + mask_full = features.new_ones( + batch_size, *spatial_size, 1, requires_grad=empty_render + ) + if not empty_render: + normals = implicit_function_gradient(differentiable_surface_points)[ + None, :, 0, : + ] + normals_full.view(-1, 3)[surface_mask] = normals + render_full.view(-1, self.render_features_dimensions)[surface_mask] = ( + self._rgb_network( + features, + differentiable_surface_points[None], + normals, + ray_bundle, + surface_mask[None, :, None], + pooling_fn=None, # TODO + ) + ) + mask_full.view(-1, 1)[~surface_mask] = torch.sigmoid( + # pyre-fixme[6]: For 1st param expected `Tensor` but got `float`. + -self.soft_mask_alpha + * sdf_output[~surface_mask] + ) + + # scatter points with surface_mask + points_full = ray_bundle.origins.detach().clone() + points_full.view(-1, 3)[surface_mask] = differentiable_surface_points + + # TODO: it is sparse here but otherwise dense + return RendererOutput( + features=render_full, + normals=normals_full, + depths=depth.reshape(batch_size, *spatial_size, 1), + masks=mask_full, # this is a differentiable approximation, see (7) in the paper + points=points_full, + aux={"grad_theta": grad_theta}, # TODO: will be moved to eikonal loss + # TODO: do we need sdf_output, grad_theta? Only for loss probably + ) + + +def _sample_network( + surface_output, + surface_sdf_values, + surface_points_grad, + surface_dists, + surface_cam_loc, + surface_ray_dirs, + eps: float = 1e-4, +): + # t -> t(theta) + surface_ray_dirs_0 = surface_ray_dirs.detach() + surface_points_dot = torch.bmm( + surface_points_grad.view(-1, 1, 3), surface_ray_dirs_0.view(-1, 3, 1) + ).squeeze(-1) + dot_sign = (surface_points_dot >= 0).to(surface_points_dot) * 2 - 1 + surface_dists_theta = surface_dists - (surface_output - surface_sdf_values) / ( + surface_points_dot.abs().clip(eps) * dot_sign + ) + + # t(theta) -> x(theta,c,v) + surface_points_theta_c_v = surface_cam_loc + surface_dists_theta * surface_ray_dirs + + return surface_points_theta_c_v + + +@torch.enable_grad() +def _gradient(module, rays_points_world): + rays_points_world.requires_grad_(True) + y = module.forward(rays_points_world=rays_points_world)[:, :1] + d_output = torch.ones_like(y, requires_grad=False, device=y.device) + gradients = torch.autograd.grad( + outputs=y, + inputs=rays_points_world, + grad_outputs=d_output, + create_graph=True, + retain_graph=True, + only_inputs=True, + )[0] + return gradients.unsqueeze(1) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/utils.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..70b2fc06a690a5be7ea1e08385b39c6a1fbacdbc --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/utils.py @@ -0,0 +1,212 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +# Note: The #noqa comments below are for unused imports of pluggable implementations +# which are part of implicitron. They ensure that the registry is prepopulated. + +import warnings +from logging import Logger +from typing import Any, Dict, Optional, Tuple + +import torch +import tqdm +from pytorch3d.common.compat import prod + +from pytorch3d.implicitron.models.renderer.base import ImplicitronRayBundle + +from pytorch3d.implicitron.tools import image_utils + +from pytorch3d.implicitron.tools.utils import cat_dataclass + + +def preprocess_input( + image_rgb: Optional[torch.Tensor], + fg_probability: Optional[torch.Tensor], + depth_map: Optional[torch.Tensor], + mask_images: bool, + mask_depths: bool, + mask_threshold: float, + bg_color: Tuple[float, float, float], +) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor], Optional[torch.Tensor]]: + """ + Helper function to preprocess the input images and optional depth maps + to apply masking if required. + + Args: + image_rgb: A tensor of shape `(B, 3, H, W)` containing a batch of rgb images + corresponding to the source viewpoints from which features will be extracted + fg_probability: A tensor of shape `(B, 1, H, W)` containing a batch + of foreground masks with values in [0, 1]. + depth_map: A tensor of shape `(B, 1, H, W)` containing a batch of depth maps. + mask_images: Whether or not to mask the RGB image background given the + foreground mask (the `fg_probability` argument of `GenericModel.forward`) + mask_depths: Whether or not to mask the depth image background given the + foreground mask (the `fg_probability` argument of `GenericModel.forward`) + mask_threshold: If greater than 0.0, the foreground mask is + thresholded by this value before being applied to the RGB/Depth images + bg_color: RGB values for setting the background color of input image + if mask_images=True. Defaults to (0.0, 0.0, 0.0). Each renderer has its own + way to determine the background color of its output, unrelated to this. + + Returns: + Modified image_rgb, fg_mask, depth_map + """ + if image_rgb is not None and image_rgb.ndim == 3: + # The FrameData object is used for both frames and batches of frames, + # and a user might get this error if those were confused. + # Perhaps a user has a FrameData `fd` representing a single frame and + # wrote something like `model(**fd)` instead of + # `model(**fd.collate([fd]))`. + raise ValueError( + "Model received unbatched inputs. " + + "Perhaps they came from a FrameData which had not been collated." + ) + + fg_mask = fg_probability + if fg_mask is not None and mask_threshold > 0.0: + # threshold masks + warnings.warn("Thresholding masks!") + fg_mask = (fg_mask >= mask_threshold).type_as(fg_mask) + + if mask_images and fg_mask is not None and image_rgb is not None: + # mask the image + warnings.warn("Masking images!") + image_rgb = image_utils.mask_background( + image_rgb, fg_mask, dim_color=1, bg_color=torch.tensor(bg_color) + ) + + if mask_depths and fg_mask is not None and depth_map is not None: + # mask the depths + assert ( + mask_threshold > 0.0 + ), "Depths should be masked only with thresholded masks" + warnings.warn("Masking depths!") + depth_map = depth_map * fg_mask + + return image_rgb, fg_mask, depth_map + + +def log_loss_weights(loss_weights: Dict[str, float], logger: Logger) -> None: + """ + Print a table of the loss weights. + """ + loss_weights_message = ( + "-------\nloss_weights:\n" + + "\n".join(f"{k:40s}: {w:1.2e}" for k, w in loss_weights.items()) + + "-------" + ) + logger.info(loss_weights_message) + + +def weighted_sum_losses( + preds: Dict[str, torch.Tensor], loss_weights: Dict[str, float] +) -> Optional[torch.Tensor]: + """ + A helper function to compute the overall loss as the dot product + of individual loss functions with the corresponding weights. + """ + losses_weighted = [ + preds[k] * float(w) + for k, w in loss_weights.items() + if (k in preds and w != 0.0) + ] + if len(losses_weighted) == 0: + warnings.warn("No main objective found.") + return None + loss = sum(losses_weighted) + assert torch.is_tensor(loss) + return loss + + +def apply_chunked(func, chunk_generator, tensor_collator): + """ + Helper function to apply a function on a sequence of + chunked inputs yielded by a generator and collate + the result. + """ + processed_chunks = [ + func(*chunk_args, **chunk_kwargs) + for chunk_args, chunk_kwargs in chunk_generator + ] + + return cat_dataclass(processed_chunks, tensor_collator) + + +def chunk_generator( + chunk_size: int, + ray_bundle: ImplicitronRayBundle, + chunked_inputs: Dict[str, torch.Tensor], + tqdm_trigger_threshold: int, + *args, + **kwargs, +): + """ + Helper function which yields chunks of rays from the + input ray_bundle, to be used when the number of rays is + large and will not fit in memory for rendering. + """ + ( + batch_size, + *spatial_dim, + n_pts_per_ray, + ) = ray_bundle.lengths.shape # B x ... x n_pts_per_ray + if n_pts_per_ray > 0 and chunk_size % n_pts_per_ray != 0: + raise ValueError( + f"chunk_size_grid ({chunk_size}) should be divisible " + f"by n_pts_per_ray ({n_pts_per_ray})" + ) + + n_rays = prod(spatial_dim) + # special handling for raytracing-based methods + n_chunks = -(-n_rays * max(n_pts_per_ray, 1) // chunk_size) + chunk_size_in_rays = -(-n_rays // n_chunks) + + iter = range(0, n_rays, chunk_size_in_rays) + if len(iter) >= tqdm_trigger_threshold: + iter = tqdm.tqdm(iter) + + def _safe_slice( + tensor: Optional[torch.Tensor], start_idx: int, end_idx: int + ) -> Any: + return tensor[start_idx:end_idx] if tensor is not None else None + + for start_idx in iter: + end_idx = min(start_idx + chunk_size_in_rays, n_rays) + bins = ( + None + if ray_bundle.bins is None + else ray_bundle.bins.reshape(batch_size, n_rays, n_pts_per_ray + 1)[ + :, start_idx:end_idx + ] + ) + pixel_radii_2d = ( + None + if ray_bundle.pixel_radii_2d is None + else ray_bundle.pixel_radii_2d.reshape(batch_size, -1, 1)[ + :, start_idx:end_idx + ] + ) + ray_bundle_chunk = ImplicitronRayBundle( + origins=ray_bundle.origins.reshape(batch_size, -1, 3)[:, start_idx:end_idx], + directions=ray_bundle.directions.reshape(batch_size, -1, 3)[ + :, start_idx:end_idx + ], + lengths=ray_bundle.lengths.reshape(batch_size, n_rays, n_pts_per_ray)[ + :, start_idx:end_idx + ], + xys=ray_bundle.xys.reshape(batch_size, -1, 2)[:, start_idx:end_idx], + bins=bins, + pixel_radii_2d=pixel_radii_2d, + camera_ids=_safe_slice(ray_bundle.camera_ids, start_idx, end_idx), + camera_counts=_safe_slice(ray_bundle.camera_counts, start_idx, end_idx), + ) + extra_args = kwargs.copy() + for k, v in chunked_inputs.items(): + extra_args[k] = v.flatten(2)[:, :, start_idx:end_idx] + yield [ray_bundle_chunk, *args], extra_args diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/view_pooler/__init__.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/view_pooler/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ac1a72bde66f104691245d2de4e83c6863718d5 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/view_pooler/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/view_pooler/feature_aggregator.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/view_pooler/feature_aggregator.py new file mode 100644 index 0000000000000000000000000000000000000000..c223be22878ef17f72a04e3766a79970c200bf61 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/view_pooler/feature_aggregator.py @@ -0,0 +1,689 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from abc import ABC, abstractmethod +from enum import Enum +from typing import Dict, Optional, Sequence, Tuple, Union + +import torch +import torch.nn.functional as F +from pytorch3d.implicitron.models.view_pooler.view_sampler import ( + cameras_points_cartesian_product, +) +from pytorch3d.implicitron.tools.config import registry, ReplaceableBase +from pytorch3d.ops import wmean +from pytorch3d.renderer.cameras import CamerasBase + + +class ReductionFunction(Enum): + AVG = "avg" # simple average + MAX = "max" # maximum + STD = "std" # standard deviation + STD_AVG = "std_avg" # average of per-dimension standard deviations + + +class FeatureAggregatorBase(ABC, ReplaceableBase): + """ + Base class for aggregating features. + + Typically, the aggregated features and their masks are output by `ViewSampler` + which samples feature tensors extracted from a set of source images. + + Settings: + exclude_target_view: If `True`/`False`, enables/disables pooling + from target view to itself. + exclude_target_view_mask_features: If `True`, + mask the features from the target view before aggregation + concatenate_output: If `True`, + concatenate the aggregated features into a single tensor, + otherwise return a dictionary mapping feature names to tensors. + """ + + exclude_target_view: bool = True + exclude_target_view_mask_features: bool = True + concatenate_output: bool = True + + @abstractmethod + def forward( + self, + feats_sampled: Dict[str, torch.Tensor], + masks_sampled: torch.Tensor, + camera: Optional[CamerasBase] = None, + pts: Optional[torch.Tensor] = None, + **kwargs, + ) -> Union[torch.Tensor, Dict[str, torch.Tensor]]: + """ + Args: + feats_sampled: A `dict` of sampled feature tensors `{f_i: t_i}`, + where each `t_i` is a tensor of shape + `(minibatch, n_source_views, n_samples, dim_i)`. + masks_sampled: A binary mask represented as a tensor of shape + `(minibatch, n_source_views, n_samples, 1)` denoting valid + sampled features. + camera: A batch of `n_source_views` `CamerasBase` objects corresponding + to the source view cameras. + pts: A tensor of shape `(minibatch, n_samples, 3)` denoting the + 3D points whose 2D projections to source views were sampled in + order to generate `feats_sampled` and `masks_sampled`. + + Returns: + feats_aggregated: If `concatenate_output==True`, a tensor + of shape `(minibatch, reduce_dim, n_samples, sum(dim_1, ... dim_N))` + containing the concatenation of the aggregated features `feats_sampled`. + `reduce_dim` depends on the specific feature aggregator + implementation and typically equals 1 or `n_source_views`. + If `concatenate_output==False`, the aggregator does not concatenate + the aggregated features and returns a dictionary of per-feature + aggregations `{f_i: t_i_aggregated}` instead. Each `t_i_aggregated` + is of shape `(minibatch, reduce_dim, n_samples, aggr_dim_i)`. + """ + raise NotImplementedError() + + @abstractmethod + def get_aggregated_feature_dim( + self, feats_or_feats_dim: Union[Dict[str, torch.Tensor], int] + ): + """ + Returns the final dimensionality of the output aggregated features. + + Args: + feats_or_feats_dim: Either a `dict` of sampled features `{f_i: t_i}` corresponding + to the `feats_sampled` argument of `forward`, + or an `int` representing the sum of dimensionalities of each `t_i`. + + Returns: + aggregated_feature_dim: The final dimensionality of the output + aggregated features. + """ + raise NotImplementedError() + + def has_aggregation(self) -> bool: + """ + Specifies whether the aggregator reduces the output `reduce_dim` dimension to 1. + + Returns: + has_aggregation: `True` if `reduce_dim==1`, else `False`. + """ + return hasattr(self, "reduction_functions") + + +@registry.register +class IdentityFeatureAggregator(torch.nn.Module, FeatureAggregatorBase): + """ + This aggregator does not perform any feature aggregation. Depending on the + settings the aggregator allows to mask target view features and concatenate + the outputs. + """ + + def get_aggregated_feature_dim( + self, feats_or_feats_dim: Union[Dict[str, torch.Tensor], int] + ): + return _get_reduction_aggregator_feature_dim(feats_or_feats_dim, []) + + def forward( + self, + feats_sampled: Dict[str, torch.Tensor], + masks_sampled: torch.Tensor, + camera: Optional[CamerasBase] = None, + pts: Optional[torch.Tensor] = None, + **kwargs, + ) -> Union[torch.Tensor, Dict[str, torch.Tensor]]: + """ + Args: + feats_sampled: A `dict` of sampled feature tensors `{f_i: t_i}`, + where each `t_i` is a tensor of shape + `(minibatch, n_source_views, n_samples, dim_i)`. + masks_sampled: A binary mask represented as a tensor of shape + `(minibatch, n_source_views, n_samples, 1)` denoting valid + sampled features. + camera: A batch of `n_source_views` `CamerasBase` objects + corresponding to the source view cameras. + pts: A tensor of shape `(minibatch, n_samples, 3)` denoting the + 3D points whose 2D projections to source views were sampled in + order to generate `feats_sampled` and `masks_sampled`. + + Returns: + feats_aggregated: If `concatenate_output==True`, a tensor + of shape `(minibatch, 1, n_samples, sum(dim_1, ... dim_N))`. + If `concatenate_output==False`, a dictionary `{f_i: t_i_aggregated}` + with each `t_i_aggregated` of shape + `(minibatch, n_source_views, n_samples, dim_i)`. + """ + if self.exclude_target_view_mask_features: + feats_sampled = _mask_target_view_features(feats_sampled) + feats_aggregated = feats_sampled + if self.concatenate_output: + feats_aggregated = torch.cat(tuple(feats_aggregated.values()), dim=-1) + return feats_aggregated + + +@registry.register +class ReductionFeatureAggregator(torch.nn.Module, FeatureAggregatorBase): + """ + Aggregates using a set of predefined `reduction_functions` and concatenates + the results of each aggregation function along the + channel dimension. The reduction functions singularize the second dimension + of the sampled features which stacks the source views. + + Settings: + reduction_functions: A list of `ReductionFunction`s` that reduce the + the stack of source-view-specific features to a single feature. + """ + + reduction_functions: Tuple[ReductionFunction, ...] = ( + ReductionFunction.AVG, + ReductionFunction.STD, + ) + + def get_aggregated_feature_dim( + self, feats_or_feats_dim: Union[Dict[str, torch.Tensor], int] + ): + return _get_reduction_aggregator_feature_dim( + feats_or_feats_dim, self.reduction_functions + ) + + def forward( + self, + feats_sampled: Dict[str, torch.Tensor], + masks_sampled: torch.Tensor, + camera: Optional[CamerasBase] = None, + pts: Optional[torch.Tensor] = None, + **kwargs, + ) -> Union[torch.Tensor, Dict[str, torch.Tensor]]: + """ + Args: + feats_sampled: A `dict` of sampled feature tensors `{f_i: t_i}`, + where each `t_i` is a tensor of shape + `(minibatch, n_source_views, n_samples, dim_i)`. + masks_sampled: A binary mask represented as a tensor of shape + `(minibatch, n_source_views, n_samples, 1)` denoting valid + sampled features. + camera: A batch of `n_source_views` `CamerasBase` objects corresponding + to the source view cameras. + pts: A tensor of shape `(minibatch, n_samples, 3)` denoting the + 3D points whose 2D projections to source views were sampled in + order to generate `feats_sampled` and `masks_sampled`. + + Returns: + feats_aggregated: If `concatenate_output==True`, a tensor + of shape `(minibatch, 1, n_samples, sum(dim_1, ... dim_N))`. + If `concatenate_output==False`, a dictionary `{f_i: t_i_aggregated}` + with each `t_i_aggregated` of shape `(minibatch, 1, n_samples, aggr_dim_i)`. + """ + + pts_batch, n_cameras = masks_sampled.shape[:2] + if self.exclude_target_view_mask_features: + feats_sampled = _mask_target_view_features(feats_sampled) + sampling_mask = _get_view_sampling_mask( + n_cameras, + pts_batch, + masks_sampled.device, + self.exclude_target_view, + ) + aggr_weigths = masks_sampled[..., 0] * sampling_mask[..., None] + feats_aggregated = { + k: _avgmaxstd_reduction_function( + f, + aggr_weigths, + dim=1, + reduction_functions=self.reduction_functions, + ) + for k, f in feats_sampled.items() + } + if self.concatenate_output: + feats_aggregated = torch.cat(tuple(feats_aggregated.values()), dim=-1) + return feats_aggregated + + +@registry.register +class AngleWeightedReductionFeatureAggregator(torch.nn.Module, FeatureAggregatorBase): + """ + Performs a weighted aggregation using a set of predefined `reduction_functions` + and concatenates the results of each aggregation function along the + channel dimension. The weights are proportional to the cosine of the + angle between the target ray and the source ray:: + + weight = ( + dot(target_ray, source_ray) * 0.5 + 0.5 + self.min_ray_angle_weight + )**self.weight_by_ray_angle_gamma + + The reduction functions singularize the second dimension + of the sampled features which stacks the source views. + + Settings: + reduction_functions: A list of `ReductionFunction`s that reduce the + the stack of source-view-specific features to a single feature. + min_ray_angle_weight: The minimum possible aggregation weight + before rasising to the power of `self.weight_by_ray_angle_gamma`. + weight_by_ray_angle_gamma: The exponent of the cosine of the ray angles + used when calculating the angle-based aggregation weights. + """ + + reduction_functions: Tuple[ReductionFunction, ...] = ( + ReductionFunction.AVG, + ReductionFunction.STD, + ) + weight_by_ray_angle_gamma: float = 1.0 + min_ray_angle_weight: float = 0.1 + + def get_aggregated_feature_dim( + self, feats_or_feats_dim: Union[Dict[str, torch.Tensor], int] + ): + return _get_reduction_aggregator_feature_dim( + feats_or_feats_dim, self.reduction_functions + ) + + def forward( + self, + feats_sampled: Dict[str, torch.Tensor], + masks_sampled: torch.Tensor, + camera: Optional[CamerasBase] = None, + pts: Optional[torch.Tensor] = None, + **kwargs, + ) -> Union[torch.Tensor, Dict[str, torch.Tensor]]: + """ + Args: + feats_sampled: A `dict` of sampled feature tensors `{f_i: t_i}`, + where each `t_i` is a tensor of shape + `(minibatch, n_source_views, n_samples, dim_i)`. + masks_sampled: A binary mask represented as a tensor of shape + `(minibatch, n_source_views, n_samples, 1)` denoting valid + sampled features. + camera: A batch of `n_source_views` `CamerasBase` objects + corresponding to the source view cameras. + pts: A tensor of shape `(minibatch, n_samples, 3)` denoting the + 3D points whose 2D projections to source views were sampled in + order to generate `feats_sampled` and `masks_sampled`. + + Returns: + feats_aggregated: If `concatenate_output==True`, a tensor + of shape `(minibatch, 1, n_samples, sum(dim_1, ... dim_N))`. + If `concatenate_output==False`, a dictionary `{f_i: t_i_aggregated}` + with each `t_i_aggregated` of shape + `(minibatch, n_source_views, n_samples, dim_i)`. + """ + + if camera is None: + raise ValueError("camera cannot be None for angle weighted aggregation") + + if pts is None: + raise ValueError("Points cannot be None for angle weighted aggregation") + + pts_batch, n_cameras = masks_sampled.shape[:2] + if self.exclude_target_view_mask_features: + feats_sampled = _mask_target_view_features(feats_sampled) + view_sampling_mask = _get_view_sampling_mask( + n_cameras, + pts_batch, + masks_sampled.device, + self.exclude_target_view, + ) + aggr_weights = _get_angular_reduction_weights( + view_sampling_mask, + masks_sampled, + camera, + pts, + self.min_ray_angle_weight, + self.weight_by_ray_angle_gamma, + ) + assert torch.isfinite(aggr_weights).all() + feats_aggregated = { + k: _avgmaxstd_reduction_function( + f, + aggr_weights, + dim=1, + reduction_functions=self.reduction_functions, + ) + for k, f in feats_sampled.items() + } + if self.concatenate_output: + feats_aggregated = torch.cat(tuple(feats_aggregated.values()), dim=-1) + return feats_aggregated + + +@registry.register +class AngleWeightedIdentityFeatureAggregator(torch.nn.Module, FeatureAggregatorBase): + """ + This aggregator does not perform any feature aggregation. It only weights + the features by the weights proportional to the cosine of the + angle between the target ray and the source ray:: + + weight = ( + dot(target_ray, source_ray) * 0.5 + 0.5 + self.min_ray_angle_weight + )**self.weight_by_ray_angle_gamma + + Settings: + min_ray_angle_weight: The minimum possible aggregation weight + before rasising to the power of `self.weight_by_ray_angle_gamma`. + weight_by_ray_angle_gamma: The exponent of the cosine of the ray angles + used when calculating the angle-based aggregation weights. + + Additionally the aggregator allows to mask target view features and to concatenate + the outputs. + """ + + weight_by_ray_angle_gamma: float = 1.0 + min_ray_angle_weight: float = 0.1 + + def get_aggregated_feature_dim( + self, feats_or_feats_dim: Union[Dict[str, torch.Tensor], int] + ): + return _get_reduction_aggregator_feature_dim(feats_or_feats_dim, []) + + def forward( + self, + feats_sampled: Dict[str, torch.Tensor], + masks_sampled: torch.Tensor, + camera: Optional[CamerasBase] = None, + pts: Optional[torch.Tensor] = None, + **kwargs, + ) -> Union[torch.Tensor, Dict[str, torch.Tensor]]: + """ + Args: + feats_sampled: A `dict` of sampled feature tensors `{f_i: t_i}`, + where each `t_i` is a tensor of shape + `(minibatch, n_source_views, n_samples, dim_i)`. + masks_sampled: A binary mask represented as a tensor of shape + `(minibatch, n_source_views, n_samples, 1)` denoting valid + sampled features. + camera: A batch of `n_source_views` `CamerasBase` objects corresponding + to the source view cameras. + pts: A tensor of shape `(minibatch, n_samples, 3)` denoting the + 3D points whose 2D projections to source views were sampled in + order to generate `feats_sampled` and `masks_sampled`. + + Returns: + feats_aggregated: If `concatenate_output==True`, a tensor + of shape `(minibatch, n_source_views, n_samples, sum(dim_1, ... dim_N))`. + If `concatenate_output==False`, a dictionary `{f_i: t_i_aggregated}` + with each `t_i_aggregated` of shape + `(minibatch, n_source_views, n_samples, dim_i)`. + """ + + if camera is None: + raise ValueError("camera cannot be None for angle weighted aggregation") + + if pts is None: + raise ValueError("Points cannot be None for angle weighted aggregation") + + pts_batch, n_cameras = masks_sampled.shape[:2] + if self.exclude_target_view_mask_features: + feats_sampled = _mask_target_view_features(feats_sampled) + view_sampling_mask = _get_view_sampling_mask( + n_cameras, + pts_batch, + masks_sampled.device, + self.exclude_target_view, + ) + aggr_weights = _get_angular_reduction_weights( + view_sampling_mask, + masks_sampled, + camera, + pts, + self.min_ray_angle_weight, + self.weight_by_ray_angle_gamma, + ) + feats_aggregated = { + k: f * aggr_weights[..., None] for k, f in feats_sampled.items() + } + if self.concatenate_output: + feats_aggregated = torch.cat(tuple(feats_aggregated.values()), dim=-1) + return feats_aggregated + + +def _get_reduction_aggregator_feature_dim( + feats_or_feats_dim: Union[Dict[str, torch.Tensor], int], + reduction_functions: Sequence[ReductionFunction], +) -> int: + if isinstance(feats_or_feats_dim, int): + feat_dim = feats_or_feats_dim + else: + feat_dim = int(sum(f.shape[1] for f in feats_or_feats_dim.values())) + if len(reduction_functions) == 0: + return feat_dim + return sum( + _get_reduction_function_output_dim( + reduction_function, + feat_dim, + ) + for reduction_function in reduction_functions + ) + + +def _get_reduction_function_output_dim( + reduction_function: ReductionFunction, + feat_dim: int, +) -> int: + if reduction_function == ReductionFunction.STD_AVG: + return 1 + else: + return feat_dim + + +def _get_view_sampling_mask( + n_cameras: int, + pts_batch: int, + device: Union[str, torch.device], + exclude_target_view: bool, +): + return ( + -torch.eye(n_cameras, device=device, dtype=torch.float32) + * float(exclude_target_view) + + 1.0 + )[:pts_batch] + + +def _mask_target_view_features( + feats_sampled: Dict[str, torch.Tensor], +): + # mask out the sampled features to be sure we dont use them + # anywhere later + one_feature_sampled = next(iter(feats_sampled.values())) + pts_batch, n_cameras = one_feature_sampled.shape[:2] + view_sampling_mask = _get_view_sampling_mask( + n_cameras, + pts_batch, + one_feature_sampled.device, + True, + ) + view_sampling_mask = view_sampling_mask.view( + pts_batch, n_cameras, *([1] * (one_feature_sampled.ndim - 2)) + ) + return {k: f * view_sampling_mask for k, f in feats_sampled.items()} + + +def _get_angular_reduction_weights( + view_sampling_mask: torch.Tensor, + masks_sampled: torch.Tensor, + camera: CamerasBase, + pts: torch.Tensor, + min_ray_angle_weight: float, + weight_by_ray_angle_gamma: float, +): + aggr_weights = masks_sampled.clone()[..., 0] + assert not any(v is None for v in [camera, pts]) + angle_weight = _get_ray_angle_weights( + camera, + pts, + min_ray_angle_weight, + weight_by_ray_angle_gamma, + ) + assert torch.isfinite(angle_weight).all() + # multiply the final aggr weights with ray angles + view_sampling_mask = view_sampling_mask.view( + *view_sampling_mask.shape[:2], *([1] * (aggr_weights.ndim - 2)) + ) + aggr_weights = ( + aggr_weights * angle_weight.reshape_as(aggr_weights) * view_sampling_mask + ) + return aggr_weights + + +def _get_ray_dir_dot_prods(camera: CamerasBase, pts: torch.Tensor): + n_cameras = camera.R.shape[0] + pts_batch = pts.shape[0] + + camera_rep, pts_rep = cameras_points_cartesian_product(camera, pts) + + # does not produce nans randomly unlike get_camera_center() below + cam_centers_rep = -torch.bmm( + camera_rep.T[:, None], + camera_rep.R.permute(0, 2, 1), + ).reshape(-1, *([1] * (pts.ndim - 2)), 3) + # cam_centers_rep = camera_rep.get_camera_center().reshape( + # -1, *([1]*(pts.ndim - 2)), 3 + # ) + + ray_dirs = F.normalize(pts_rep - cam_centers_rep, dim=-1) + # camera_rep = [ pts_rep = [ + # camera[0] pts[0], + # camera[0] pts[1], + # camera[0] ..., + # ... pts[batch_pts-1], + # camera[1] pts[0], + # camera[1] pts[1], + # camera[1] ..., + # ... pts[batch_pts-1], + # ... ..., + # camera[n_cameras-1] pts[0], + # camera[n_cameras-1] pts[1], + # camera[n_cameras-1] ..., + # ... pts[batch_pts-1], + # ] ] + + ray_dirs_reshape = ray_dirs.view(n_cameras, pts_batch, -1, 3) + # [ + # [pts_0 in cam_0, pts_1 in cam_0, ..., pts_m in cam_0], + # [pts_0 in cam_1, pts_1 in cam_1, ..., pts_m in cam_1], + # ... + # [pts_0 in cam_n, pts_1 in cam_n, ..., pts_m in cam_n], + # ] + + ray_dirs_pts = torch.stack([ray_dirs_reshape[i, i] for i in range(pts_batch)]) + ray_dir_dot_prods = (ray_dirs_pts[None] * ray_dirs_reshape).sum( + dim=-1 + ) # pts_batch x n_cameras x n_pts + + return ray_dir_dot_prods.transpose(0, 1) + + +def _get_ray_angle_weights( + camera: CamerasBase, + pts: torch.Tensor, + min_ray_angle_weight: float, + weight_by_ray_angle_gamma: float, +): + ray_dir_dot_prods = _get_ray_dir_dot_prods( + camera, pts + ) # pts_batch x n_cameras x ... x 3 + angle_weight_01 = ray_dir_dot_prods * 0.5 + 0.5 # [-1, 1] to [0, 1] + angle_weight = (angle_weight_01 + min_ray_angle_weight) ** weight_by_ray_angle_gamma + return angle_weight + + +def _avgmaxstd_reduction_function( + x: torch.Tensor, + w: torch.Tensor, + reduction_functions: Sequence[ReductionFunction], + dim: int = 1, +): + """ + Args: + x: Features to aggreagate. Tensor of shape `(batch, n_views, ..., dim)`. + w: Aggregation weights. Tensor of shape `(batch, n_views, ...,)`. + dim: the dimension along which to aggregate. + reduction_functions: The set of reduction functions. + + Returns: + x_aggr: Aggregation of `x` to a tensor of shape `(batch, 1, ..., dim_aggregate)`. + """ + + pooled_features = [] + + mu = None + std = None + + if ReductionFunction.AVG in reduction_functions: + # average pool + mu = _avg_reduction_function(x, w, dim=dim) + pooled_features.append(mu) + + if ReductionFunction.STD in reduction_functions: + # standard-dev pool + std = _std_reduction_function(x, w, dim=dim, mu=mu) + pooled_features.append(std) + + if ReductionFunction.STD_AVG in reduction_functions: + # average-of-standard-dev pool + stdavg = _std_avg_reduction_function(x, w, dim=dim, mu=mu, std=std) + pooled_features.append(stdavg) + + if ReductionFunction.MAX in reduction_functions: + max_ = _max_reduction_function(x, w, dim=dim) + pooled_features.append(max_) + + # cat all results along the feature dimension (the last dim) + x_aggr = torch.cat(pooled_features, dim=-1) + + # zero out features that were all masked out + # pyre-fixme[16]: `bool` has no attribute `type_as`. + any_active = (w.max(dim=dim, keepdim=True).values > 1e-4).type_as(x_aggr) + x_aggr = x_aggr * any_active[..., None] + + # some asserts to check that everything was done right + assert torch.isfinite(x_aggr).all() + assert x_aggr.shape[1] == 1 + + return x_aggr + + +def _avg_reduction_function( + x: torch.Tensor, + w: torch.Tensor, + dim: int = 1, +): + mu = wmean(x, w, dim=dim, eps=1e-2) + return mu + + +def _std_reduction_function( + x: torch.Tensor, + w: torch.Tensor, + dim: int = 1, + mu: Optional[torch.Tensor] = None, # pre-computed mean +): + if mu is None: + mu = _avg_reduction_function(x, w, dim=dim) + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and `int`. + std = wmean((x - mu) ** 2, w, dim=dim, eps=1e-2).clamp(1e-4).sqrt() + # FIXME: somehow this is extremely heavy in mem? + return std + + +def _std_avg_reduction_function( + x: torch.Tensor, + w: torch.Tensor, + dim: int = 1, + mu: Optional[torch.Tensor] = None, # pre-computed mean + std: Optional[torch.Tensor] = None, # pre-computed std +): + if std is None: + std = _std_reduction_function(x, w, dim=dim, mu=mu) + stdmean = std.mean(dim=-1, keepdim=True) + return stdmean + + +def _max_reduction_function( + x: torch.Tensor, + w: torch.Tensor, + dim: int = 1, + big_M_factor: float = 10.0, +): + big_M = x.max(dim=dim, keepdim=True).values.abs() * big_M_factor + max_ = (x * w - ((1 - w) * big_M)).max(dim=dim, keepdim=True).values + return max_ diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/view_pooler/view_pooler.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/view_pooler/view_pooler.py new file mode 100644 index 0000000000000000000000000000000000000000..ec6c4666edc9db6870f5d9c233bc67ebe18465cc --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/view_pooler/view_pooler.py @@ -0,0 +1,129 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from typing import Dict, List, Optional, Union + +import torch +from pytorch3d.implicitron.tools.config import Configurable, run_auto_creation +from pytorch3d.renderer.cameras import CamerasBase + +from .feature_aggregator import FeatureAggregatorBase +from .view_sampler import ViewSampler + + +# pyre-ignore: 13 +class ViewPooler(Configurable, torch.nn.Module): + """ + Implements sampling of image-based features at the 2d projections of a set + of 3D points, and a subsequent aggregation of the resulting set of features + per-point. + + Args: + view_sampler: An instance of ViewSampler which is used for sampling of + image-based features at the 2D projections of a set + of 3D points. + feature_aggregator_class_type: The name of the feature aggregator class which + is available in the global registry. + feature_aggregator: A feature aggregator class which inherits from + FeatureAggregatorBase. Typically, the aggregated features and their + masks are output by a `ViewSampler` which samples feature tensors extracted + from a set of source images. FeatureAggregator executes step (4) above. + """ + + view_sampler: ViewSampler + feature_aggregator_class_type: str = "AngleWeightedReductionFeatureAggregator" + feature_aggregator: FeatureAggregatorBase + + def __post_init__(self): + run_auto_creation(self) + + def get_aggregated_feature_dim(self, feats: Union[Dict[str, torch.Tensor], int]): + """ + Returns the final dimensionality of the output aggregated features. + + Args: + feats: Either a `dict` of sampled features `{f_i: t_i}` corresponding + to the `feats_sampled` argument of `feature_aggregator,forward`, + or an `int` representing the sum of dimensionalities of each `t_i`. + + Returns: + aggregated_feature_dim: The final dimensionality of the output + aggregated features. + """ + return self.feature_aggregator.get_aggregated_feature_dim(feats) + + def has_aggregation(self): + """ + Specifies whether the `feature_aggregator` reduces the output `reduce_dim` + dimension to 1. + + Returns: + has_aggregation: `True` if `reduce_dim==1`, else `False`. + """ + return self.feature_aggregator.has_aggregation() + + def forward( + self, + *, # force kw args + pts: torch.Tensor, + seq_id_pts: Union[List[int], List[str], torch.LongTensor], + camera: CamerasBase, + seq_id_camera: Union[List[int], List[str], torch.LongTensor], + feats: Dict[str, torch.Tensor], + masks: Optional[torch.Tensor], + **kwargs, + ) -> Union[torch.Tensor, Dict[str, torch.Tensor]]: + """ + Project each point cloud from a batch of point clouds to corresponding + input cameras, sample features at the 2D projection locations in a batch + of source images, and aggregate the pointwise sampled features. + + Args: + pts: A tensor of shape `[pts_batch x n_pts x 3]` in world coords. + seq_id_pts: LongTensor of shape `[pts_batch]` denoting the ids of the scenes + from which `pts` were extracted, or a list of string names. + camera: 'n_cameras' cameras, each coresponding to a batch element of `feats`. + seq_id_camera: LongTensor of shape `[n_cameras]` denoting the ids of the scenes + corresponding to cameras in `camera`, or a list of string names. + feats: a dict of tensors of per-image features `{feat_i: T_i}`. + Each tensor `T_i` is of shape `[n_cameras x dim_i x H_i x W_i]`. + masks: `[n_cameras x 1 x H x W]`, define valid image regions + for sampling `feats`. + Returns: + feats_aggregated: If `feature_aggregator.concatenate_output==True`, a tensor + of shape `(pts_batch, reduce_dim, n_pts, sum(dim_1, ... dim_N))` + containing the aggregated features. `reduce_dim` depends on + the specific feature aggregator implementation and typically + equals 1 or `n_cameras`. + If `feature_aggregator.concatenate_output==False`, the aggregator + does not concatenate the aggregated features and returns a dictionary + of per-feature aggregations `{f_i: t_i_aggregated}` instead. + Each `t_i_aggregated` is of shape + `(pts_batch, reduce_dim, n_pts, aggr_dim_i)`. + """ + + # (1) Sample features and masks at the ray points + sampled_feats, sampled_masks = self.view_sampler( + pts=pts, + seq_id_pts=seq_id_pts, + camera=camera, + seq_id_camera=seq_id_camera, + feats=feats, + masks=masks, + ) + + # (2) Aggregate features from multiple views + # pyre-fixme[29]: `Union[torch.Tensor, torch.nn.Module]` is not a function. + feats_aggregated = self.feature_aggregator( # noqa: E731 + sampled_feats, + sampled_masks, + pts=pts, + camera=camera, + ) # TODO: do we need to pass a callback rather than compute here? + + return feats_aggregated diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/view_pooler/view_sampler.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/view_pooler/view_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..dabb8b7fc359f993845381ef40a729d30aeb8ec6 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/view_pooler/view_sampler.py @@ -0,0 +1,295 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from typing import Dict, List, Optional, Tuple, Union + +import torch +from pytorch3d.implicitron.tools.config import Configurable +from pytorch3d.renderer.cameras import CamerasBase +from pytorch3d.renderer.utils import ndc_grid_sample + + +class ViewSampler(Configurable, torch.nn.Module): + """ + Implements sampling of image-based features at the 2d projections of a set + of 3D points. + + Args: + masked_sampling: If `True`, the `sampled_masks` output of `self.forward` + contains the input `masks` sampled at the 2d projections. Otherwise, + all entries of `sampled_masks` are set to 1. + sampling_mode: Controls the mode of the `torch.nn.functional.grid_sample` + function used to interpolate the sampled feature tensors at the + locations of the 2d projections. + """ + + masked_sampling: bool = False + sampling_mode: str = "bilinear" + + def forward( + self, + *, # force kw args + pts: torch.Tensor, + seq_id_pts: Union[List[int], List[str], torch.LongTensor], + camera: CamerasBase, + seq_id_camera: Union[List[int], List[str], torch.LongTensor], + feats: Dict[str, torch.Tensor], + masks: Optional[torch.Tensor], + **kwargs, + ) -> Tuple[Dict[str, torch.Tensor], torch.Tensor]: + """ + Project each point cloud from a batch of point clouds to corresponding + input cameras and sample features at the 2D projection locations. + + Args: + pts: A tensor of shape `[pts_batch x n_pts x 3]` in world coords. + seq_id_pts: LongTensor of shape `[pts_batch]` denoting the ids of the scenes + from which `pts` were extracted, or a list of string names. + camera: 'n_cameras' cameras, each coresponding to a batch element of `feats`. + seq_id_camera: LongTensor of shape `[n_cameras]` denoting the ids of the scenes + corresponding to cameras in `camera`, or a list of string names. + feats: a dict of tensors of per-image features `{feat_i: T_i}`. + Each tensor `T_i` is of shape `[n_cameras x dim_i x H_i x W_i]`. + masks: `[n_cameras x 1 x H x W]`, define valid image regions + for sampling `feats`. + Returns: + sampled_feats: Dict of sampled features `{feat_i: sampled_T_i}`. + Each `sampled_T_i` of shape `[pts_batch, n_cameras, n_pts, dim_i]`. + sampled_masks: A tensor with mask of the sampled features + of shape `(pts_batch, n_cameras, n_pts, 1)`. + """ + + # convert sequence ids to long tensors + seq_id_pts, seq_id_camera = [ + handle_seq_id(seq_id, pts.device) for seq_id in [seq_id_pts, seq_id_camera] + ] + + if self.masked_sampling and masks is None: + raise ValueError( + "Masks have to be provided for `self.masked_sampling==True`" + ) + + # project pts to all cameras and sample feats from the locations of + # the 2D projections + sampled_feats_all_cams, sampled_masks_all_cams = project_points_and_sample( + pts, + feats, + camera, + masks if self.masked_sampling else None, + sampling_mode=self.sampling_mode, + ) + + # generate the mask that invalidates features sampled from + # non-corresponding cameras + camera_pts_mask = (seq_id_camera[None] == seq_id_pts[:, None])[ + ..., None, None + ].to(pts) + + # mask the sampled features and masks + sampled_feats = { + k: f * camera_pts_mask for k, f in sampled_feats_all_cams.items() + } + sampled_masks = sampled_masks_all_cams * camera_pts_mask + + return sampled_feats, sampled_masks + + +def project_points_and_sample( + pts: torch.Tensor, + feats: Dict[str, torch.Tensor], + camera: CamerasBase, + masks: Optional[torch.Tensor], + eps: float = 1e-2, + sampling_mode: str = "bilinear", +) -> Tuple[Dict[str, torch.Tensor], torch.Tensor]: + """ + Project each point cloud from a batch of point clouds to all input cameras + and sample features at the 2D projection locations. + + Args: + pts: `(pts_batch, n_pts, 3)` tensor containing a batch of 3D point clouds. + feats: A dict `{feat_i: feat_T_i}` of features to sample, + where each `feat_T_i` is a tensor of shape + `(n_cameras, feat_i_dim, feat_i_H, feat_i_W)` + of `feat_i_dim`-dimensional features extracted from `n_cameras` + source views. + camera: A batch of `n_cameras` cameras corresponding to their feature + tensors `feat_T_i` from `feats`. + masks: A tensor of shape `(n_cameras, 1, mask_H, mask_W)` denoting + valid locations for sampling. + eps: A small constant controlling the minimum depth of projections + of `pts` to avoid divisons by zero in the projection operation. + sampling_mode: Sampling mode of the grid sampler. + + Returns: + sampled_feats: Dict of sampled features `{feat_i: sampled_T_i}`. + Each `sampled_T_i` is of shape + `(pts_batch, n_cameras, n_pts, feat_i_dim)`. + sampled_masks: A tensor with the mask of the sampled features + of shape `(pts_batch, n_cameras, n_pts, 1)`. + If `masks` is `None`, the returned `sampled_masks` will be + filled with 1s. + """ + + n_cameras = camera.R.shape[0] + pts_batch = pts.shape[0] + n_pts = pts.shape[1:-1] + + camera_rep, pts_rep = cameras_points_cartesian_product(camera, pts) + + # The eps here is super-important to avoid NaNs in backprop! + proj_rep = camera_rep.transform_points( + pts_rep.reshape(n_cameras * pts_batch, -1, 3), eps=eps + )[..., :2] + # [ pts1 in cam1, pts2 in cam1, pts3 in cam1, + # pts1 in cam2, pts2 in cam2, pts3 in cam2, + # pts1 in cam3, pts2 in cam3, pts3 in cam3 ] + + # reshape for the grid sampler + sampling_grid_ndc = proj_rep.view(n_cameras, pts_batch, -1, 2) + # [ [pts1 in cam1, pts2 in cam1, pts3 in cam1], + # [pts1 in cam2, pts2 in cam2, pts3 in cam2], + # [pts1 in cam3, pts2 in cam3, pts3 in cam3] ] + # n_cameras x pts_batch x n_pts x 2 + + # sample both feats + feats_sampled = { + k: ndc_grid_sample( + f, + sampling_grid_ndc, + mode=sampling_mode, + align_corners=False, + ) + .permute(2, 0, 3, 1) + .reshape(pts_batch, n_cameras, *n_pts, -1) + for k, f in feats.items() + } # {k: pts_batch x n_cameras x *n_pts x dim} for each feat type "k" + + if masks is not None: + # sample masks + masks_sampled = ( + ndc_grid_sample( + masks, + sampling_grid_ndc, + mode=sampling_mode, + align_corners=False, + ) + .permute(2, 0, 3, 1) + .reshape(pts_batch, n_cameras, *n_pts, 1) + ) + else: + masks_sampled = sampling_grid_ndc.new_ones(pts_batch, n_cameras, *n_pts, 1) + + return feats_sampled, masks_sampled + + +def handle_seq_id( + seq_id: Union[torch.LongTensor, List[str], List[int]], + device, +) -> torch.LongTensor: + """ + Converts the input sequence id to a LongTensor. + + Args: + seq_id: A sequence of sequence ids. + device: The target device of the output. + Returns + long_seq_id: `seq_id` converted to a `LongTensor` and moved to `device`. + """ + if not torch.is_tensor(seq_id): + if isinstance(seq_id[0], str): + seq_id = [hash(s) for s in seq_id] + # pyre-fixme[9]: seq_id has type `Union[List[int], List[str], LongTensor]`; + # used as `Tensor`. + seq_id = torch.tensor(seq_id, dtype=torch.long, device=device) + # pyre-fixme[16]: Item `List` of `Union[List[int], List[str], LongTensor]` has + # no attribute `to`. + return seq_id.to(device) + + +def cameras_points_cartesian_product( + camera: CamerasBase, pts: torch.Tensor +) -> Tuple[CamerasBase, torch.Tensor]: + """ + Generates all pairs of pairs of elements from 'camera' and 'pts' and returns + `camera_rep` and `pts_rep` such that:: + + camera_rep = [ pts_rep = [ + camera[0] pts[0], + camera[0] pts[1], + camera[0] ..., + ... pts[batch_pts-1], + camera[1] pts[0], + camera[1] pts[1], + camera[1] ..., + ... pts[batch_pts-1], + ... ..., + camera[n_cameras-1] pts[0], + camera[n_cameras-1] pts[1], + camera[n_cameras-1] ..., + ... pts[batch_pts-1], + ] ] + + Args: + camera: A batch of `n_cameras` cameras. + pts: A batch of `batch_pts` points of shape `(batch_pts, ..., dim)` + + Returns: + camera_rep: A batch of batch_pts*n_cameras cameras such that:: + + camera_rep = [ + camera[0] + camera[0] + camera[0] + ... + camera[1] + camera[1] + camera[1] + ... + ... + camera[n_cameras-1] + camera[n_cameras-1] + camera[n_cameras-1] + ] + + + pts_rep: Repeated `pts` of shape `(batch_pts*n_cameras, ..., dim)`, + such that:: + + pts_rep = [ + pts[0], + pts[1], + ..., + pts[batch_pts-1], + pts[0], + pts[1], + ..., + pts[batch_pts-1], + ..., + pts[0], + pts[1], + ..., + pts[batch_pts-1], + ] + + """ + n_cameras = camera.R.shape[0] + batch_pts = pts.shape[0] + pts_rep = pts.repeat(n_cameras, *[1 for _ in pts.shape[1:]]) + idx_cams = ( + torch.arange(n_cameras)[:, None] + .expand( + n_cameras, + batch_pts, + ) + .reshape(batch_pts * n_cameras) + ) + # pyre-fixme[6]: For 1st param expected `Union[List[int], int, LongTensor]` but + # got `Tensor`. + camera_rep = camera[idx_cams] + return camera_rep, pts_rep diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/visualization/__init__.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/visualization/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6267ddbf92b36f9ebed9b194fd0eab924ad35556 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/visualization/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/visualization/render_flyaround.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/visualization/render_flyaround.py new file mode 100644 index 0000000000000000000000000000000000000000..44ca3de5bbb610deda34c7e736d4b86e7f27de11 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/models/visualization/render_flyaround.py @@ -0,0 +1,393 @@ +#!/usr/bin/env python3 +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +import logging +import math +import os +import random +from typing import ( + Any, + Dict, + Iterable, + List, + Optional, + Sequence, + Tuple, + TYPE_CHECKING, + Union, +) + +import numpy as np +import torch +import torch.nn.functional as Fu +from pytorch3d.implicitron.dataset.dataset_base import DatasetBase, FrameData +from pytorch3d.implicitron.dataset.utils import is_train_frame +from pytorch3d.implicitron.models.base_model import EvaluationMode +from pytorch3d.implicitron.tools.eval_video_trajectory import ( + generate_eval_video_cameras, +) +from pytorch3d.implicitron.tools.video_writer import VideoWriter +from pytorch3d.implicitron.tools.vis_utils import ( + get_visdom_connection, + make_depth_image, +) +from tqdm import tqdm + +if TYPE_CHECKING: + from visdom import Visdom + +logger = logging.getLogger(__name__) + + +def render_flyaround( + dataset: DatasetBase, + sequence_name: str, + model: torch.nn.Module, + output_video_path: str, + n_flyaround_poses: int = 40, + fps: int = 20, + trajectory_type: str = "circular_lsq_fit", + max_angle: float = 2 * math.pi, + trajectory_scale: float = 1.1, + scene_center: Tuple[float, float, float] = (0.0, 0.0, 0.0), + up: Tuple[float, float, float] = (0.0, -1.0, 0.0), + traj_offset: float = 0.0, + n_source_views: int = 9, + visdom_show_preds: bool = False, + visdom_environment: str = "render_flyaround", + visdom_server: str = "http://127.0.0.1", + visdom_port: int = 8097, + num_workers: int = 10, + device: Union[str, torch.device] = "cuda", + seed: Optional[int] = None, + video_resize: Optional[Tuple[int, int]] = None, + output_video_frames_dir: Optional[str] = None, + visualize_preds_keys: Sequence[str] = ( + "images_render", + "masks_render", + "depths_render", + "_all_source_images", + ), +) -> None: + """ + Uses `model` to generate a video consisting of renders of a scene imaged from + a camera flying around the scene. The scene is specified with the `dataset` object and + `sequence_name` which denotes the name of the scene whose frames are in `dataset`. + + Args: + dataset: The dataset object containing frames from a sequence in `sequence_name`. + sequence_name: Name of a sequence from `dataset`. + model: The model whose predictions are going to be visualized. + output_video_path: The path to the video output by this script. + n_flyaround_poses: The number of camera poses of the flyaround trajectory. + fps: Framerate of the output video. + trajectory_type: The type of the camera trajectory. Can be one of: + circular_lsq_fit: Camera centers follow a trajectory obtained + by fitting a 3D circle to train_cameras centers. + All cameras are looking towards scene_center. + figure_eight: Figure-of-8 trajectory around the center of the + central camera of the training dataset. + trefoil_knot: Same as 'figure_eight', but the trajectory has a shape + of a trefoil knot (https://en.wikipedia.org/wiki/Trefoil_knot). + figure_eight_knot: Same as 'figure_eight', but the trajectory has a shape + of a figure-eight knot + (https://en.wikipedia.org/wiki/Figure-eight_knot_(mathematics)). + trajectory_type: The type of the camera trajectory. Can be one of: + circular_lsq_fit: Camera centers follow a trajectory obtained + by fitting a 3D circle to train_cameras centers. + All cameras are looking towards scene_center. + figure_eight: Figure-of-8 trajectory around the center of the + central camera of the training dataset. + trefoil_knot: Same as 'figure_eight', but the trajectory has a shape + of a trefoil knot (https://en.wikipedia.org/wiki/Trefoil_knot). + figure_eight_knot: Same as 'figure_eight', but the trajectory has a shape + of a figure-eight knot + (https://en.wikipedia.org/wiki/Figure-eight_knot_(mathematics)). + max_angle: Defines the total length of the generated camera trajectory. + All possible trajectories (set with the `trajectory_type` argument) are + periodic with the period of `time==2pi`. + E.g. setting `trajectory_type=circular_lsq_fit` and `time=4pi` will generate + a trajectory of camera poses rotating the total of 720 deg around the object. + trajectory_scale: The extent of the trajectory. + scene_center: The center of the scene in world coordinates which all + the cameras from the generated trajectory look at. + up: The "up" vector of the scene (=the normal of the scene floor). + Active for the `trajectory_type="circular"`. + traj_offset: 3D offset vector added to each point of the trajectory. + n_source_views: The number of source views sampled from the known views of the + training sequence added to each evaluation batch. + visdom_show_preds: If `True`, exports the visualizations to visdom. + visdom_environment: The name of the visdom environment. + visdom_server: The address of the visdom server. + visdom_port: The visdom port. + num_workers: The number of workers used to load the training data. + seed: The random seed used for reproducible sampling of the source views. + video_resize: Optionally, defines the size of the output video. + output_video_frames_dir: If specified, the frames of the output video are going + to be permanently stored in this directory. + visualize_preds_keys: The names of the model predictions to visualize. + """ + + if seed is None: + seed = hash(sequence_name) + + if visdom_show_preds: + viz = get_visdom_connection(server=visdom_server, port=visdom_port) + else: + viz = None + + logger.info(f"Loading all data of sequence '{sequence_name}'.") + seq_idx = list(dataset.sequence_indices_in_order(sequence_name)) + train_data = _load_whole_dataset(dataset, seq_idx, num_workers=num_workers) + assert all(train_data.sequence_name[0] == sn for sn in train_data.sequence_name) + # pyre-ignore[6] + sequence_set_name = "train" if is_train_frame(train_data.frame_type)[0] else "test" + logger.info(f"Sequence set = {sequence_set_name}.") + train_cameras = train_data.camera + time = torch.linspace(0, max_angle, n_flyaround_poses + 1)[:n_flyaround_poses] + test_cameras = generate_eval_video_cameras( + train_cameras, + time=time, + n_eval_cams=n_flyaround_poses, + trajectory_type=trajectory_type, + trajectory_scale=trajectory_scale, + scene_center=scene_center, + up=up, + focal_length=None, + principal_point=torch.zeros(n_flyaround_poses, 2), + traj_offset_canonical=(0.0, 0.0, traj_offset), + ) + + # sample the source views reproducibly + with torch.random.fork_rng(): + torch.manual_seed(seed) + source_views_i = torch.randperm(len(seq_idx))[:n_source_views] + + # add the first dummy view that will get replaced with the target camera + source_views_i = Fu.pad(source_views_i, [1, 0]) + source_views = [seq_idx[i] for i in source_views_i.tolist()] + batch = _load_whole_dataset(dataset, source_views, num_workers=num_workers) + assert all(batch.sequence_name[0] == sn for sn in batch.sequence_name) + + preds_total = [] + for n in tqdm(range(n_flyaround_poses), total=n_flyaround_poses): + # set the first batch camera to the target camera + for k in ("R", "T", "focal_length", "principal_point"): + getattr(batch.camera, k)[0] = getattr(test_cameras[n], k) + + # Move to cuda + net_input = batch.to(device) + with torch.no_grad(): + preds = model(**{**net_input, "evaluation_mode": EvaluationMode.EVALUATION}) + + # make sure we dont overwrite something + assert all(k not in preds for k in net_input.keys()) + preds.update(net_input) # merge everything into one big dict + + # Render the predictions to images + rendered_pred = _images_from_preds(preds, extract_keys=visualize_preds_keys) + preds_total.append(rendered_pred) + + # show the preds every 5% of the export iterations + if visdom_show_preds and ( + n % max(n_flyaround_poses // 20, 1) == 0 or n == n_flyaround_poses - 1 + ): + assert viz is not None + _show_predictions( + preds_total, + sequence_name=batch.sequence_name[0], + viz=viz, + viz_env=visdom_environment, + predicted_keys=visualize_preds_keys, + ) + + logger.info(f"Exporting videos for sequence {sequence_name} ...") + _generate_prediction_videos( + preds_total, + sequence_name=batch.sequence_name[0], + viz=viz, + viz_env=visdom_environment, + fps=fps, + video_path=output_video_path, + resize=video_resize, + video_frames_dir=output_video_frames_dir, + predicted_keys=visualize_preds_keys, + ) + + +def _load_whole_dataset( + dataset: torch.utils.data.Dataset, idx: Sequence[int], num_workers: int = 10 +) -> FrameData: + load_all_dataloader = torch.utils.data.DataLoader( + torch.utils.data.Subset(dataset, idx), + batch_size=len(idx), + num_workers=num_workers, + shuffle=False, + collate_fn=FrameData.collate, + ) + return next(iter(load_all_dataloader)) + + +def _images_from_preds( + preds: Dict[str, Any], + extract_keys: Iterable[str] = ( + "image_rgb", + "images_render", + "fg_probability", + "masks_render", + "depths_render", + "depth_map", + "_all_source_images", + ), +) -> Dict[str, torch.Tensor]: + imout = {} + for k in extract_keys: + if k == "_all_source_images" and "image_rgb" in preds: + src_ims = preds["image_rgb"][1:].cpu().detach().clone() + v = _stack_images(src_ims, None)[None] + else: + if k not in preds or preds[k] is None: + print(f"cant show {k}") + continue + v = preds[k].cpu().detach().clone() + if k.startswith("depth"): + mask_resize = Fu.interpolate( + preds["masks_render"], + size=preds[k].shape[2:], + mode="nearest", + ) + v = make_depth_image(preds[k], mask_resize) + if v.shape[1] == 1: + v = v.repeat(1, 3, 1, 1) + imout[k] = v.detach().cpu() + + return imout + + +def _stack_images(ims: torch.Tensor, size: Optional[Tuple[int, int]]) -> torch.Tensor: + ba = ims.shape[0] + H = int(np.ceil(np.sqrt(ba))) + W = H + n_add = H * W - ba + if n_add > 0: + ims = torch.cat((ims, torch.zeros_like(ims[:1]).repeat(n_add, 1, 1, 1))) + + ims = ims.view(H, W, *ims.shape[1:]) + cated = torch.cat([torch.cat(list(row), dim=2) for row in ims], dim=1) + if size is not None: + cated = Fu.interpolate(cated[None], size=size, mode="bilinear")[0] + return cated.clamp(0.0, 1.0) + + +def _show_predictions( + preds: List[Dict[str, Any]], + sequence_name: str, + viz: "Visdom", + viz_env: str = "visualizer", + predicted_keys: Sequence[str] = ( + "images_render", + "masks_render", + "depths_render", + "_all_source_images", + ), + n_samples=10, + one_image_width=200, +) -> None: + """Given a list of predictions visualize them into a single image using visdom.""" + assert isinstance(preds, list) + + pred_all = [] + # Randomly choose a subset of the rendered images, sort by ordr in the sequence + n_samples = min(n_samples, len(preds)) + pred_idx = sorted(random.sample(list(range(len(preds))), n_samples)) + for predi in pred_idx: + # Make the concatentation for the same camera vertically + pred_all.append( + torch.cat( + [ + torch.nn.functional.interpolate( + preds[predi][k].cpu(), + scale_factor=one_image_width / preds[predi][k].shape[3], + mode="bilinear", + ).clamp(0.0, 1.0) + for k in predicted_keys + ], + dim=2, + ) + ) + # Concatenate the images horizontally + pred_all_cat = torch.cat(pred_all, dim=3)[0] + viz.image( + pred_all_cat, + win="show_predictions", + env=viz_env, + opts={"title": f"pred_{sequence_name}"}, + ) + + +def _generate_prediction_videos( + preds: List[Dict[str, Any]], + sequence_name: str, + viz: Optional["Visdom"] = None, + viz_env: str = "visualizer", + predicted_keys: Sequence[str] = ( + "images_render", + "masks_render", + "depths_render", + "_all_source_images", + ), + fps: int = 20, + video_path: str = "/tmp/video", + video_frames_dir: Optional[str] = None, + resize: Optional[Tuple[int, int]] = None, +) -> None: + """Given a list of predictions create and visualize rotating videos of the + objects using visdom. + """ + + # make sure the target video directory exists + os.makedirs(os.path.dirname(video_path), exist_ok=True) + + # init a video writer for each predicted key + vws = {} + for k in predicted_keys: + if k not in preds[0]: + logger.warn(f"Cannot generate video for prediction key '{k}'") + continue + cache_dir = ( + None + if video_frames_dir is None + else os.path.join(video_frames_dir, f"{sequence_name}_{k}") + ) + vws[k] = VideoWriter( + fps=fps, + out_path=f"{video_path}_{sequence_name}_{k}.mp4", + cache_dir=cache_dir, + ) + + for rendered_pred in tqdm(preds): + for k in vws: + vws[k].write_frame( + rendered_pred[k][0].clip(0.0, 1.0).detach().cpu().numpy(), + resize=resize, + ) + + for k in predicted_keys: + if k not in vws: + continue + vws[k].get_video() + logger.info(f"Generated {vws[k].out_path}.") + if viz is not None: + viz.video( + videofile=vws[k].out_path, + env=viz_env, + win=k, # we reuse the same window otherwise visdom dies + opts={"title": sequence_name + " " + k}, + ) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/third_party/__init__.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/third_party/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ac1a72bde66f104691245d2de4e83c6863718d5 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/third_party/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/third_party/hyperlayers.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/third_party/hyperlayers.py new file mode 100644 index 0000000000000000000000000000000000000000..8ad9ebeb73e938b019b358b265b9013fac9434d5 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/third_party/hyperlayers.py @@ -0,0 +1,255 @@ +# a copy-paste from https://github.com/vsitzmann/scene-representation-networks/blob/master/hyperlayers.py +# fmt: off +# flake8: noqa + +# pyre-unsafe +'''Pytorch implementations of hyper-network modules. +''' +import functools + +import torch +import torch.nn as nn + +from . import pytorch_prototyping + + +def partialclass(cls, *args, **kwds): + class NewCls(cls): + __init__ = functools.partialmethod(cls.__init__, *args, **kwds) + + return NewCls + + +class LookupLayer(nn.Module): + def __init__(self, in_ch, out_ch, num_objects): + super().__init__() + + self.out_ch = out_ch + self.lookup_lin = LookupLinear(in_ch, out_ch, num_objects=num_objects) + self.norm_nl = nn.Sequential( + nn.LayerNorm([self.out_ch], elementwise_affine=False), nn.ReLU(inplace=True) + ) + + def forward(self, obj_idx): + net = nn.Sequential(self.lookup_lin(obj_idx), self.norm_nl) + return net + + +class LookupFC(nn.Module): + def __init__( + self, + hidden_ch, + num_hidden_layers, + num_objects, + in_ch, + out_ch, + outermost_linear=False, + ): + super().__init__() + self.layers = nn.ModuleList() + self.layers.append( + LookupLayer(in_ch=in_ch, out_ch=hidden_ch, num_objects=num_objects) + ) + + for i in range(num_hidden_layers): + self.layers.append( + LookupLayer(in_ch=hidden_ch, out_ch=hidden_ch, num_objects=num_objects) + ) + + if outermost_linear: + self.layers.append( + LookupLinear(in_ch=hidden_ch, out_ch=out_ch, num_objects=num_objects) + ) + else: + self.layers.append( + LookupLayer(in_ch=hidden_ch, out_ch=out_ch, num_objects=num_objects) + ) + + def forward(self, obj_idx): + net = [] + for i in range(len(self.layers)): + net.append(self.layers[i](obj_idx)) + + return nn.Sequential(*net) + + +class LookupLinear(nn.Module): + def __init__(self, in_ch, out_ch, num_objects): + super().__init__() + self.in_ch = in_ch + self.out_ch = out_ch + + self.hypo_params = nn.Embedding(num_objects, in_ch * out_ch + out_ch) + + for i in range(num_objects): + nn.init.kaiming_normal_( + self.hypo_params.weight.data[i, : self.in_ch * self.out_ch].view( + self.out_ch, self.in_ch + ), + a=0.0, + nonlinearity="relu", + mode="fan_in", + ) + self.hypo_params.weight.data[i, self.in_ch * self.out_ch :].fill_(0.0) + + def forward(self, obj_idx): + hypo_params = self.hypo_params(obj_idx) + + # Indices explicit to catch erros in shape of output layer + weights = hypo_params[..., : self.in_ch * self.out_ch] + biases = hypo_params[ + ..., self.in_ch * self.out_ch : (self.in_ch * self.out_ch) + self.out_ch + ] + + biases = biases.view(*(biases.size()[:-1]), 1, self.out_ch) + weights = weights.view(*(weights.size()[:-1]), self.out_ch, self.in_ch) + + return BatchLinear(weights=weights, biases=biases) + + +class HyperLayer(nn.Module): + """A hypernetwork that predicts a single Dense Layer, including LayerNorm and a ReLU.""" + + def __init__( + self, in_ch, out_ch, hyper_in_ch, hyper_num_hidden_layers, hyper_hidden_ch + ): + super().__init__() + + self.hyper_linear = HyperLinear( + in_ch=in_ch, + out_ch=out_ch, + hyper_in_ch=hyper_in_ch, + hyper_num_hidden_layers=hyper_num_hidden_layers, + hyper_hidden_ch=hyper_hidden_ch, + ) + self.norm_nl = nn.Sequential( + nn.LayerNorm([out_ch], elementwise_affine=False), nn.ReLU(inplace=True) + ) + + def forward(self, hyper_input): + """ + :param hyper_input: input to hypernetwork. + :return: nn.Module; predicted fully connected network. + """ + return nn.Sequential(self.hyper_linear(hyper_input), self.norm_nl) + + +class HyperFC(nn.Module): + """Builds a hypernetwork that predicts a fully connected neural network.""" + + def __init__( + self, + hyper_in_ch, + hyper_num_hidden_layers, + hyper_hidden_ch, + hidden_ch, + num_hidden_layers, + in_ch, + out_ch, + outermost_linear=False, + ): + super().__init__() + + PreconfHyperLinear = partialclass( + HyperLinear, + hyper_in_ch=hyper_in_ch, + hyper_num_hidden_layers=hyper_num_hidden_layers, + hyper_hidden_ch=hyper_hidden_ch, + ) + PreconfHyperLayer = partialclass( + HyperLayer, + hyper_in_ch=hyper_in_ch, + hyper_num_hidden_layers=hyper_num_hidden_layers, + hyper_hidden_ch=hyper_hidden_ch, + ) + + self.layers = nn.ModuleList() + self.layers.append(PreconfHyperLayer(in_ch=in_ch, out_ch=hidden_ch)) + + for i in range(num_hidden_layers): + self.layers.append(PreconfHyperLayer(in_ch=hidden_ch, out_ch=hidden_ch)) + + if outermost_linear: + self.layers.append(PreconfHyperLinear(in_ch=hidden_ch, out_ch=out_ch)) + else: + self.layers.append(PreconfHyperLayer(in_ch=hidden_ch, out_ch=out_ch)) + + def forward(self, hyper_input): + """ + :param hyper_input: Input to hypernetwork. + :return: nn.Module; Predicted fully connected neural network. + """ + net = [] + for i in range(len(self.layers)): + net.append(self.layers[i](hyper_input)) + + return nn.Sequential(*net) + + +class BatchLinear(nn.Module): + def __init__(self, weights, biases): + """Implements a batch linear layer. + + :param weights: Shape: (batch, out_ch, in_ch) + :param biases: Shape: (batch, 1, out_ch) + """ + super().__init__() + + self.weights = weights + self.biases = biases + + def __repr__(self): + return "BatchLinear(in_ch=%d, out_ch=%d)" % ( + self.weights.shape[-1], + self.weights.shape[-2], + ) + + def forward(self, input): + output = input.matmul( + self.weights.permute( + *[i for i in range(len(self.weights.shape) - 2)], -1, -2 + ) + ) + output += self.biases + return output + + +def last_hyper_layer_init(m) -> None: + if type(m) == nn.Linear: + nn.init.kaiming_normal_(m.weight, a=0.0, nonlinearity="relu", mode="fan_in") + m.weight.data *= 1e-1 + + +class HyperLinear(nn.Module): + """A hypernetwork that predicts a single linear layer (weights & biases).""" + + def __init__( + self, in_ch, out_ch, hyper_in_ch, hyper_num_hidden_layers, hyper_hidden_ch + ): + + super().__init__() + self.in_ch = in_ch + self.out_ch = out_ch + + self.hypo_params = pytorch_prototyping.FCBlock( + in_features=hyper_in_ch, + hidden_ch=hyper_hidden_ch, + num_hidden_layers=hyper_num_hidden_layers, + out_features=(in_ch * out_ch) + out_ch, + outermost_linear=True, + ) + self.hypo_params[-1].apply(last_hyper_layer_init) + + def forward(self, hyper_input): + hypo_params = self.hypo_params(hyper_input) + + # Indices explicit to catch erros in shape of output layer + weights = hypo_params[..., : self.in_ch * self.out_ch] + biases = hypo_params[ + ..., self.in_ch * self.out_ch : (self.in_ch * self.out_ch) + self.out_ch + ] + + biases = biases.view(*(biases.size()[:-1]), 1, self.out_ch) + weights = weights.view(*(weights.size()[:-1]), self.out_ch, self.in_ch) + + return BatchLinear(weights=weights, biases=biases) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/third_party/pytorch_prototyping.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/third_party/pytorch_prototyping.py new file mode 100644 index 0000000000000000000000000000000000000000..6e2366d545fae2306428c6c578fa29b7ec4537cb --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/third_party/pytorch_prototyping.py @@ -0,0 +1,773 @@ +# a copy-paste from https://raw.githubusercontent.com/vsitzmann/pytorch_prototyping/10f49b1e7df38a58fd78451eac91d7ac1a21df64/pytorch_prototyping.py +# fmt: off +# flake8: noqa + +# pyre-unsafe +'''A number of custom pytorch modules with sane defaults that I find useful for model prototyping. +''' +import torch +import torch.nn as nn +import torchvision.utils +from torch.nn import functional as F + + +class FCLayer(nn.Module): + def __init__(self, in_features, out_features): + super().__init__() + self.net = nn.Sequential( + nn.Linear(in_features, out_features), + nn.LayerNorm([out_features]), + nn.ReLU(inplace=True), + ) + + def forward(self, input): + return self.net(input) + + +# From https://gist.github.com/wassname/ecd2dac6fc8f9918149853d17e3abf02 +class LayerNormConv2d(nn.Module): + def __init__(self, num_features, eps=1e-5, affine=True): + super().__init__() + self.num_features = num_features + self.affine = affine + self.eps = eps + + if self.affine: + self.gamma = nn.Parameter(torch.Tensor(num_features).uniform_()) + self.beta = nn.Parameter(torch.zeros(num_features)) + + def forward(self, x): + shape = [-1] + [1] * (x.dim() - 1) + mean = x.view(x.size(0), -1).mean(1).view(*shape) + std = x.view(x.size(0), -1).std(1).view(*shape) + + y = (x - mean) / (std + self.eps) + if self.affine: + shape = [1, -1] + [1] * (x.dim() - 2) + y = self.gamma.view(*shape) * y + self.beta.view(*shape) + return y + + +class FCBlock(nn.Module): + def __init__( + self, + hidden_ch, + num_hidden_layers, + in_features, + out_features, + outermost_linear=False, + ): + super().__init__() + + self.net = [] + self.net.append(FCLayer(in_features=in_features, out_features=hidden_ch)) + + for i in range(num_hidden_layers): + self.net.append(FCLayer(in_features=hidden_ch, out_features=hidden_ch)) + + if outermost_linear: + self.net.append(nn.Linear(in_features=hidden_ch, out_features=out_features)) + else: + self.net.append(FCLayer(in_features=hidden_ch, out_features=out_features)) + + self.net = nn.Sequential(*self.net) + self.net.apply(self.init_weights) + + def __getitem__(self, item): + return self.net[item] + + def init_weights(self, m): + if type(m) == nn.Linear: + nn.init.kaiming_normal_(m.weight, a=0.0, nonlinearity="relu", mode="fan_in") + + def forward(self, input): + return self.net(input) + + +class DownBlock3D(nn.Module): + """A 3D convolutional downsampling block.""" + + def __init__(self, in_channels, out_channels, norm=nn.BatchNorm3d): + super().__init__() + + self.net = [ + nn.ReplicationPad3d(1), + nn.Conv3d( + in_channels, + out_channels, + kernel_size=4, + padding=0, + stride=2, + bias=False if norm is not None else True, + ), + ] + + if norm is not None: + self.net += [norm(out_channels, affine=True)] + + self.net += [nn.LeakyReLU(0.2, True)] + self.net = nn.Sequential(*self.net) + + def forward(self, x): + return self.net(x) + + +class UpBlock3D(nn.Module): + """A 3D convolutional upsampling block.""" + + def __init__(self, in_channels, out_channels, norm=nn.BatchNorm3d): + super().__init__() + + self.net = [ + nn.ConvTranspose3d( + in_channels, + out_channels, + kernel_size=4, + stride=2, + padding=1, + bias=False if norm is not None else True, + ), + ] + + if norm is not None: + self.net += [norm(out_channels, affine=True)] + + self.net += [nn.ReLU(True)] + self.net = nn.Sequential(*self.net) + + def forward(self, x, skipped=None): + if skipped is not None: + input = torch.cat([skipped, x], dim=1) + else: + input = x + return self.net(input) + + +class Conv3dSame(torch.nn.Module): + """3D convolution that pads to keep spatial dimensions equal. + Cannot deal with stride. Only quadratic kernels (=scalar kernel_size). + """ + + def __init__( + self, + in_channels, + out_channels, + kernel_size, + bias=True, + padding_layer=nn.ReplicationPad3d, + ): + """ + :param in_channels: Number of input channels + :param out_channels: Number of output channels + :param kernel_size: Scalar. Spatial dimensions of kernel (only quadratic kernels supported). + :param bias: Whether or not to use bias. + :param padding_layer: Which padding to use. Default is reflection padding. + """ + super().__init__() + ka = kernel_size // 2 + kb = ka - 1 if kernel_size % 2 == 0 else ka + self.net = nn.Sequential( + padding_layer((ka, kb, ka, kb, ka, kb)), + nn.Conv3d(in_channels, out_channels, kernel_size, bias=bias, stride=1), + ) + + def forward(self, x): + return self.net(x) + + +class Conv2dSame(torch.nn.Module): + """2D convolution that pads to keep spatial dimensions equal. + Cannot deal with stride. Only quadratic kernels (=scalar kernel_size). + """ + + def __init__( + self, + in_channels, + out_channels, + kernel_size, + bias=True, + padding_layer=nn.ReflectionPad2d, + ): + """ + :param in_channels: Number of input channels + :param out_channels: Number of output channels + :param kernel_size: Scalar. Spatial dimensions of kernel (only quadratic kernels supported). + :param bias: Whether or not to use bias. + :param padding_layer: Which padding to use. Default is reflection padding. + """ + super().__init__() + ka = kernel_size // 2 + kb = ka - 1 if kernel_size % 2 == 0 else ka + self.net = nn.Sequential( + padding_layer((ka, kb, ka, kb)), + nn.Conv2d(in_channels, out_channels, kernel_size, bias=bias, stride=1), + ) + + self.weight = self.net[1].weight + self.bias = self.net[1].bias + + def forward(self, x): + return self.net(x) + + +class UpBlock(nn.Module): + """A 2d-conv upsampling block with a variety of options for upsampling, and following best practices / with + reasonable defaults. (LeakyReLU, kernel size multiple of stride) + """ + + def __init__( + self, + in_channels, + out_channels, + post_conv=True, + use_dropout=False, + dropout_prob=0.1, + norm=nn.BatchNorm2d, + upsampling_mode="transpose", + ): + """ + :param in_channels: Number of input channels + :param out_channels: Number of output channels + :param post_conv: Whether to have another convolutional layer after the upsampling layer. + :param use_dropout: bool. Whether to use dropout or not. + :param dropout_prob: Float. The dropout probability (if use_dropout is True) + :param norm: Which norm to use. If None, no norm is used. Default is Batchnorm with affinity. + :param upsampling_mode: Which upsampling mode: + transpose: Upsampling with stride-2, kernel size 4 transpose convolutions. + bilinear: Feature map is upsampled with bilinear upsampling, then a conv layer. + nearest: Feature map is upsampled with nearest neighbor upsampling, then a conv layer. + shuffle: Feature map is upsampled with pixel shuffling, then a conv layer. + """ + super().__init__() + + net = list() + + if upsampling_mode == "transpose": + net += [ + nn.ConvTranspose2d( + in_channels, + out_channels, + kernel_size=4, + stride=2, + padding=1, + bias=True if norm is None else False, + ) + ] + elif upsampling_mode == "bilinear": + net += [nn.UpsamplingBilinear2d(scale_factor=2)] + net += [ + Conv2dSame( + in_channels, + out_channels, + kernel_size=3, + bias=True if norm is None else False, + ) + ] + elif upsampling_mode == "nearest": + net += [nn.UpsamplingNearest2d(scale_factor=2)] + net += [ + Conv2dSame( + in_channels, + out_channels, + kernel_size=3, + bias=True if norm is None else False, + ) + ] + elif upsampling_mode == "shuffle": + net += [nn.PixelShuffle(upscale_factor=2)] + net += [ + Conv2dSame( + in_channels // 4, + out_channels, + kernel_size=3, + bias=True if norm is None else False, + ) + ] + else: + raise ValueError("Unknown upsampling mode!") + + if norm is not None: + net += [norm(out_channels, affine=True)] + + net += [nn.ReLU(True)] + + if use_dropout: + net += [nn.Dropout2d(dropout_prob, False)] + + if post_conv: + net += [ + Conv2dSame( + out_channels, + out_channels, + kernel_size=3, + bias=True if norm is None else False, + ) + ] + + if norm is not None: + net += [norm(out_channels, affine=True)] + + net += [nn.ReLU(True)] + + if use_dropout: + net += [nn.Dropout2d(0.1, False)] + + self.net = nn.Sequential(*net) + + def forward(self, x, skipped=None): + if skipped is not None: + input = torch.cat([skipped, x], dim=1) + else: + input = x + return self.net(input) + + +class DownBlock(nn.Module): + """A 2D-conv downsampling block following best practices / with reasonable defaults + (LeakyReLU, kernel size multiple of stride) + """ + + def __init__( + self, + in_channels, + out_channels, + prep_conv=True, + middle_channels=None, + use_dropout=False, + dropout_prob=0.1, + norm=nn.BatchNorm2d, + ): + """ + :param in_channels: Number of input channels + :param out_channels: Number of output channels + :param prep_conv: Whether to have another convolutional layer before the downsampling layer. + :param middle_channels: If prep_conv is true, this sets the number of channels between the prep and downsampling + convs. + :param use_dropout: bool. Whether to use dropout or not. + :param dropout_prob: Float. The dropout probability (if use_dropout is True) + :param norm: Which norm to use. If None, no norm is used. Default is Batchnorm with affinity. + """ + super().__init__() + + if middle_channels is None: + middle_channels = in_channels + + net = list() + + if prep_conv: + net += [ + nn.ReflectionPad2d(1), + nn.Conv2d( + in_channels, + middle_channels, + kernel_size=3, + padding=0, + stride=1, + bias=True if norm is None else False, + ), + ] + + if norm is not None: + net += [norm(middle_channels, affine=True)] + + net += [nn.LeakyReLU(0.2, True)] + + if use_dropout: + net += [nn.Dropout2d(dropout_prob, False)] + + net += [ + nn.ReflectionPad2d(1), + nn.Conv2d( + middle_channels, + out_channels, + kernel_size=4, + padding=0, + stride=2, + bias=True if norm is None else False, + ), + ] + + if norm is not None: + net += [norm(out_channels, affine=True)] + + net += [nn.LeakyReLU(0.2, True)] + + if use_dropout: + net += [nn.Dropout2d(dropout_prob, False)] + + self.net = nn.Sequential(*net) + + def forward(self, x): + return self.net(x) + + +class Unet3d(nn.Module): + """A 3d-Unet implementation with sane defaults.""" + + def __init__( + self, + in_channels, + out_channels, + nf0, + num_down, + max_channels, + norm=nn.BatchNorm3d, + outermost_linear=False, + ): + """ + :param in_channels: Number of input channels + :param out_channels: Number of output channels + :param nf0: Number of features at highest level of U-Net + :param num_down: Number of downsampling stages. + :param max_channels: Maximum number of channels (channels multiply by 2 with every downsampling stage) + :param norm: Which norm to use. If None, no norm is used. Default is Batchnorm with affinity. + :param outermost_linear: Whether the output layer should be a linear layer or a nonlinear one. + """ + super().__init__() + + assert num_down > 0, "Need at least one downsampling layer in UNet3d." + + # Define the in block + self.in_layer = [Conv3dSame(in_channels, nf0, kernel_size=3, bias=False)] + + if norm is not None: + self.in_layer += [norm(nf0, affine=True)] + + self.in_layer += [nn.LeakyReLU(0.2, True)] + self.in_layer = nn.Sequential(*self.in_layer) + + # Define the center UNet block. The feature map has height and width 1 --> no batchnorm. + self.unet_block = UnetSkipConnectionBlock3d( + int(min(2 ** (num_down - 1) * nf0, max_channels)), + int(min(2 ** (num_down - 1) * nf0, max_channels)), + norm=None, + ) + for i in list(range(0, num_down - 1))[::-1]: + self.unet_block = UnetSkipConnectionBlock3d( + int(min(2 ** i * nf0, max_channels)), + int(min(2 ** (i + 1) * nf0, max_channels)), + submodule=self.unet_block, + norm=norm, + ) + + # Define the out layer. Each unet block concatenates its inputs with its outputs - so the output layer + # automatically receives the output of the in_layer and the output of the last unet layer. + self.out_layer = [ + Conv3dSame(2 * nf0, out_channels, kernel_size=3, bias=outermost_linear) + ] + + if not outermost_linear: + if norm is not None: + self.out_layer += [norm(out_channels, affine=True)] + self.out_layer += [nn.ReLU(True)] + self.out_layer = nn.Sequential(*self.out_layer) + + def forward(self, x): + in_layer = self.in_layer(x) + unet = self.unet_block(in_layer) + out_layer = self.out_layer(unet) + return out_layer + + +class UnetSkipConnectionBlock3d(nn.Module): + """Helper class for building a 3D unet.""" + + def __init__(self, outer_nc, inner_nc, norm=nn.BatchNorm3d, submodule=None): + super().__init__() + + if submodule is None: + model = [ + DownBlock3D(outer_nc, inner_nc, norm=norm), + UpBlock3D(inner_nc, outer_nc, norm=norm), + ] + else: + model = [ + DownBlock3D(outer_nc, inner_nc, norm=norm), + submodule, + UpBlock3D(2 * inner_nc, outer_nc, norm=norm), + ] + + self.model = nn.Sequential(*model) + + def forward(self, x): + forward_passed = self.model(x) + return torch.cat([x, forward_passed], 1) + + +class UnetSkipConnectionBlock(nn.Module): + """Helper class for building a 2D unet.""" + + def __init__( + self, + outer_nc, + inner_nc, + upsampling_mode, + norm=nn.BatchNorm2d, + submodule=None, + use_dropout=False, + dropout_prob=0.1, + ): + super().__init__() + + if submodule is None: + model = [ + DownBlock( + outer_nc, + inner_nc, + use_dropout=use_dropout, + dropout_prob=dropout_prob, + norm=norm, + ), + UpBlock( + inner_nc, + outer_nc, + use_dropout=use_dropout, + dropout_prob=dropout_prob, + norm=norm, + upsampling_mode=upsampling_mode, + ), + ] + else: + model = [ + DownBlock( + outer_nc, + inner_nc, + use_dropout=use_dropout, + dropout_prob=dropout_prob, + norm=norm, + ), + submodule, + UpBlock( + 2 * inner_nc, + outer_nc, + use_dropout=use_dropout, + dropout_prob=dropout_prob, + norm=norm, + upsampling_mode=upsampling_mode, + ), + ] + + self.model = nn.Sequential(*model) + + def forward(self, x): + forward_passed = self.model(x) + return torch.cat([x, forward_passed], 1) + + +class Unet(nn.Module): + """A 2d-Unet implementation with sane defaults.""" + + def __init__( + self, + in_channels, + out_channels, + nf0, + num_down, + max_channels, + use_dropout, + upsampling_mode="transpose", + dropout_prob=0.1, + norm=nn.BatchNorm2d, + outermost_linear=False, + ): + """ + :param in_channels: Number of input channels + :param out_channels: Number of output channels + :param nf0: Number of features at highest level of U-Net + :param num_down: Number of downsampling stages. + :param max_channels: Maximum number of channels (channels multiply by 2 with every downsampling stage) + :param use_dropout: Whether to use dropout or no. + :param dropout_prob: Dropout probability if use_dropout=True. + :param upsampling_mode: Which type of upsampling should be used. See "UpBlock" for documentation. + :param norm: Which norm to use. If None, no norm is used. Default is Batchnorm with affinity. + :param outermost_linear: Whether the output layer should be a linear layer or a nonlinear one. + """ + super().__init__() + + assert num_down > 0, "Need at least one downsampling layer in UNet." + + # Define the in block + self.in_layer = [ + Conv2dSame( + in_channels, nf0, kernel_size=3, bias=True if norm is None else False + ) + ] + if norm is not None: + self.in_layer += [norm(nf0, affine=True)] + self.in_layer += [nn.LeakyReLU(0.2, True)] + + if use_dropout: + self.in_layer += [nn.Dropout2d(dropout_prob)] + self.in_layer = nn.Sequential(*self.in_layer) + + # Define the center UNet block + self.unet_block = UnetSkipConnectionBlock( + min(2 ** (num_down - 1) * nf0, max_channels), + min(2 ** (num_down - 1) * nf0, max_channels), + use_dropout=use_dropout, + dropout_prob=dropout_prob, + norm=None, # Innermost has no norm (spatial dimension 1) + upsampling_mode=upsampling_mode, + ) + + for i in list(range(0, num_down - 1))[::-1]: + self.unet_block = UnetSkipConnectionBlock( + min(2 ** i * nf0, max_channels), + min(2 ** (i + 1) * nf0, max_channels), + use_dropout=use_dropout, + dropout_prob=dropout_prob, + submodule=self.unet_block, + norm=norm, + upsampling_mode=upsampling_mode, + ) + + # Define the out layer. Each unet block concatenates its inputs with its outputs - so the output layer + # automatically receives the output of the in_layer and the output of the last unet layer. + self.out_layer = [ + Conv2dSame( + 2 * nf0, + out_channels, + kernel_size=3, + bias=outermost_linear or (norm is None), + ) + ] + + if not outermost_linear: + if norm is not None: + self.out_layer += [norm(out_channels, affine=True)] + self.out_layer += [nn.ReLU(True)] + + if use_dropout: + self.out_layer += [nn.Dropout2d(dropout_prob)] + self.out_layer = nn.Sequential(*self.out_layer) + + self.out_layer_weight = self.out_layer[0].weight + + def forward(self, x): + in_layer = self.in_layer(x) + unet = self.unet_block(in_layer) + out_layer = self.out_layer(unet) + return out_layer + + +class Identity(nn.Module): + """Helper module to allow Downsampling and Upsampling nets to default to identity if they receive an empty list.""" + + def __init__(self): + super().__init__() + + def forward(self, input): + return input + + +class DownsamplingNet(nn.Module): + """A subnetwork that downsamples a 2D feature map with strided convolutions.""" + + def __init__( + self, + per_layer_out_ch, + in_channels, + use_dropout, + dropout_prob=0.1, + last_layer_one=False, + norm=nn.BatchNorm2d, + ): + """ + :param per_layer_out_ch: python list of integers. Defines the number of output channels per layer. Length of + list defines number of downsampling steps (each step dowsamples by factor of 2.) + :param in_channels: Number of input channels. + :param use_dropout: Whether or not to use dropout. + :param dropout_prob: Dropout probability. + :param last_layer_one: Whether the output of the last layer will have a spatial size of 1. In that case, + the last layer will not have batchnorm, else, it will. + :param norm: Which norm to use. Defaults to BatchNorm. + """ + super().__init__() + + if not len(per_layer_out_ch): + self.downs = Identity() + else: + self.downs = list() + self.downs.append( + DownBlock( + in_channels, + per_layer_out_ch[0], + use_dropout=use_dropout, + dropout_prob=dropout_prob, + middle_channels=per_layer_out_ch[0], + norm=norm, + ) + ) + for i in range(0, len(per_layer_out_ch) - 1): + if last_layer_one and (i == len(per_layer_out_ch) - 2): + norm = None + self.downs.append( + DownBlock( + per_layer_out_ch[i], + per_layer_out_ch[i + 1], + dropout_prob=dropout_prob, + use_dropout=use_dropout, + norm=norm, + ) + ) + self.downs = nn.Sequential(*self.downs) + + def forward(self, input): + return self.downs(input) + + +class UpsamplingNet(nn.Module): + """A subnetwork that upsamples a 2D feature map with a variety of upsampling options.""" + + def __init__( + self, + per_layer_out_ch, + in_channels, + upsampling_mode, + use_dropout, + dropout_prob=0.1, + first_layer_one=False, + norm=nn.BatchNorm2d, + ): + """ + :param per_layer_out_ch: python list of integers. Defines the number of output channels per layer. Length of + list defines number of upsampling steps (each step upsamples by factor of 2.) + :param in_channels: Number of input channels. + :param upsampling_mode: Mode of upsampling. For documentation, see class "UpBlock" + :param use_dropout: Whether or not to use dropout. + :param dropout_prob: Dropout probability. + :param first_layer_one: Whether the input to the last layer will have a spatial size of 1. In that case, + the first layer will not have a norm, else, it will. + :param norm: Which norm to use. Defaults to BatchNorm. + """ + super().__init__() + + if not len(per_layer_out_ch): + self.ups = Identity() + else: + self.ups = list() + self.ups.append( + UpBlock( + in_channels, + per_layer_out_ch[0], + use_dropout=use_dropout, + dropout_prob=dropout_prob, + norm=None if first_layer_one else norm, + upsampling_mode=upsampling_mode, + ) + ) + for i in range(0, len(per_layer_out_ch) - 1): + self.ups.append( + UpBlock( + per_layer_out_ch[i], + per_layer_out_ch[i + 1], + use_dropout=use_dropout, + dropout_prob=dropout_prob, + norm=norm, + upsampling_mode=upsampling_mode, + ) + ) + self.ups = nn.Sequential(*self.ups) + + def forward(self, input): + return self.ups(input) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/__init__.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ac1a72bde66f104691245d2de4e83c6863718d5 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/camera_utils.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/camera_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0c76373dda4a47fc3ffe445a4b0b2136954dc140 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/camera_utils.py @@ -0,0 +1,144 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +# TODO: all this potentially goes to PyTorch3D + +import math +from typing import Tuple + +import pytorch3d as pt3d +import torch +from pytorch3d.renderer.cameras import CamerasBase + + +def jitter_extrinsics( + R: torch.Tensor, + T: torch.Tensor, + max_angle: float = (math.pi * 2.0), + translation_std: float = 1.0, + scale_std: float = 0.3, +): + """ + Jitter the extrinsic camera parameters `R` and `T` with a random similarity + transformation. The transformation rotates by a random angle between [0, max_angle]; + scales by a random factor exp(N(0, scale_std)), where N(0, scale_std) is + a random sample from a normal distrubtion with zero mean and variance scale_std; + and translates by a 3D offset sampled from N(0, translation_std). + """ + assert all(x >= 0.0 for x in (max_angle, translation_std, scale_std)) + N = R.shape[0] + R_jit = pt3d.transforms.random_rotations(1, device=R.device) + R_jit = pt3d.transforms.so3_exponential_map( + pt3d.transforms.so3_log_map(R_jit) * max_angle + ) + T_jit = torch.randn_like(R_jit[:1, :, 0]) * translation_std + rigid_transform = pt3d.ops.eyes(dim=4, N=N, device=R.device) + rigid_transform[:, :3, :3] = R_jit.expand(N, 3, 3) + rigid_transform[:, 3, :3] = T_jit.expand(N, 3) + scale_jit = torch.exp(torch.randn_like(T_jit[:, 0]) * scale_std).expand(N) + return apply_camera_alignment(R, T, rigid_transform, scale_jit) + + +def apply_camera_alignment( + R: torch.Tensor, + T: torch.Tensor, + rigid_transform: torch.Tensor, + scale: torch.Tensor, +): + """ + Args: + R: Camera rotation matrix of shape (N, 3, 3). + T: Camera translation of shape (N, 3). + rigid_transform: A tensor of shape (N, 4, 4) representing a batch of + N 4x4 tensors that map the scene pointcloud from misaligned coords + to the aligned space. + scale: A list of N scaling factors. A tensor of shape (N,) + + Returns: + R_aligned: The aligned rotations R. + T_aligned: The aligned translations T. + """ + R_rigid = rigid_transform[:, :3, :3] + T_rigid = rigid_transform[:, 3:, :3] + R_aligned = R_rigid.permute(0, 2, 1).bmm(R) + T_aligned = scale[:, None] * (T - (T_rigid @ R_aligned)[:, 0]) + return R_aligned, T_aligned + + +def get_min_max_depth_bounds(cameras, scene_center, scene_extent): + """ + Estimate near/far depth plane as: + near = dist(cam_center, self.scene_center) - self.scene_extent + far = dist(cam_center, self.scene_center) + self.scene_extent + """ + cam_center = cameras.get_camera_center() + center_dist = ( + ((cam_center - scene_center.to(cameras.R)[None]) ** 2) + .sum(dim=-1) + .clamp(0.001) + .sqrt() + ) + center_dist = center_dist.clamp(scene_extent + 1e-3) + min_depth = center_dist - scene_extent + max_depth = center_dist + scene_extent + return min_depth, max_depth + + +def volumetric_camera_overlaps( + cameras: CamerasBase, + scene_extent: float = 8.0, + scene_center: Tuple[float, float, float] = (0.0, 0.0, 0.0), + resol: int = 16, + weigh_by_ray_angle: bool = True, +): + """ + Compute the overlaps between viewing frustrums of all pairs of cameras + in `cameras`. + """ + device = cameras.device + ba = cameras.R.shape[0] + n_vox = int(resol**3) + grid = pt3d.structures.Volumes( + densities=torch.zeros([1, 1, resol, resol, resol], device=device), + volume_translation=-torch.FloatTensor(scene_center)[None].to(device), + voxel_size=2.0 * scene_extent / resol, + ).get_coord_grid(world_coordinates=True) + + grid = grid.view(1, n_vox, 3).expand(ba, n_vox, 3) + gridp = cameras.transform_points(grid, eps=1e-2) + proj_in_camera = ( + torch.prod((gridp[..., :2].abs() <= 1.0), dim=-1) + * (gridp[..., 2] > 0.0).float() + ) # ba x n_vox + + if weigh_by_ray_angle: + rays = torch.nn.functional.normalize( + grid - cameras.get_camera_center()[:, None], dim=-1 + ) + rays_masked = rays * proj_in_camera[..., None] + + # - slow and readable: + # inter = torch.zeros(ba, ba) + # for i1 in range(ba): + # for i2 in range(ba): + # inter[i1, i2] = ( + # 1 + (rays_masked[i1] * rays_masked[i2] + # ).sum(dim=-1)).sum() + + # - fast: + rays_masked = rays_masked.view(ba, n_vox * 3) + inter = n_vox + (rays_masked @ rays_masked.t()) + + else: + inter = proj_in_camera @ proj_in_camera.t() + + mass = torch.diag(inter) + iou = inter / (mass[:, None] + mass[None, :] - inter).clamp(0.1) + + return iou diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/circle_fitting.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/circle_fitting.py new file mode 100644 index 0000000000000000000000000000000000000000..7f6cd1a05ee2f0a691fa1540f5dc454282e7662c --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/circle_fitting.py @@ -0,0 +1,240 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import warnings +from dataclasses import dataclass +from math import pi +from typing import Optional + +import torch + + +def get_rotation_to_best_fit_xy( + points: torch.Tensor, centroid: Optional[torch.Tensor] = None +) -> torch.Tensor: + """ + Returns a rotation R such that `points @ R` has a best fit plane + parallel to the xy plane + + Args: + points: (*, N, 3) tensor of points in 3D + centroid: (*, 1, 3), (3,) or scalar: their centroid + + Returns: + (*, 3, 3) tensor rotation matrix + """ + if centroid is None: + centroid = points.mean(dim=-2, keepdim=True) + + points_centered = points - centroid + _, evec = torch.linalg.eigh(points_centered.transpose(-1, -2) @ points_centered) + # in general, evec can form either right- or left-handed basis, + # but we need the former to have a proper rotation (not reflection) + return torch.cat( + (evec[..., 1:], torch.cross(evec[..., 1], evec[..., 2])[..., None]), dim=-1 + ) + + +def _signed_area(path: torch.Tensor) -> torch.Tensor: + """ + Calculates the signed area / Lévy area of a 2D path. If the path is closed, + i.e. ends where it starts, this is the integral of the winding number over + the whole plane. If not, consider a closed path made by adding a straight + line from the end to the start; the signed area is the integral of the + winding number (also over the plane) with respect to that closed path. + + If this number is positive, it indicates in some sense that the path + turns anticlockwise more than clockwise, and vice versa. + + Args: + path: N x 2 tensor of points. + + Returns: + signed area, shape () + """ + # This calculation is a sum of areas of triangles of the form + # (path[0], path[i], path[i+1]), where each triangle is half a + # parallelogram. + x, y = (path[1:] - path[:1]).unbind(1) + return (y[1:] * x[:-1] - x[1:] * y[:-1]).sum() * 0.5 + + +@dataclass(frozen=True) +class Circle2D: + """ + Contains details of a circle in a plane. + Members + center: tensor shape (2,) + radius: tensor shape () + generated_points: points around the circle, shape (n_points, 2) + """ + + center: torch.Tensor + radius: torch.Tensor + generated_points: torch.Tensor + + +def fit_circle_in_2d( + points2d, *, n_points: int = 0, angles: Optional[torch.Tensor] = None +) -> Circle2D: + """ + Simple best fitting of a circle to 2D points. In particular, the circle which + minimizes the sum of the squares of the squared-distances to the circle. + + Finds (a,b) and r to minimize the sum of squares (over the x,y pairs) of + r**2 - [(x-a)**2+(y-b)**2] + i.e. + (2*a)*x + (2*b)*y + (r**2 - a**2 - b**2)*1 - (x**2 + y**2) + + In addition, generates points along the circle. If angles is None (default) + then n_points around the circle equally spaced are given. These begin at the + point closest to the first input point. They continue in the direction which + seems to match the movement of points in points2d, as judged by its + signed area. If `angles` are provided, then n_points is ignored, and points + along the circle at the given angles are returned, with the starting point + and direction as before. + + (Note that `generated_points` is affected by the order of the points in + points2d, but the other outputs are not.) + + Args: + points2d: N x 2 tensor of 2D points + n_points: number of points to generate on the circle, if angles not given + angles: optional angles in radians of points to generate. + + Returns: + Circle2D object + """ + design = torch.cat([points2d, torch.ones_like(points2d[:, :1])], dim=1) + rhs = (points2d**2).sum(1) + n_provided = points2d.shape[0] + if n_provided < 3: + raise ValueError(f"{n_provided} points are not enough to determine a circle") + solution = torch.linalg.lstsq(design, rhs[:, None]).solution + center = solution[:2, 0] / 2 + radius = torch.sqrt(solution[2, 0] + (center**2).sum()) + if n_points > 0: + if angles is not None: + warnings.warn("n_points ignored because angles provided") + else: + angles = torch.linspace(0, 2 * pi, n_points, device=points2d.device) + + if angles is not None: + initial_direction_xy = (points2d[0] - center).unbind() + initial_angle = torch.atan2(initial_direction_xy[1], initial_direction_xy[0]) + with torch.no_grad(): + anticlockwise = _signed_area(points2d) > 0 + if anticlockwise: + use_angles = initial_angle + angles + else: + use_angles = initial_angle - angles + generated_points = center[None] + radius * torch.stack( + [torch.cos(use_angles), torch.sin(use_angles)], dim=-1 + ) + else: + generated_points = points2d.new_zeros(0, 2) + return Circle2D(center=center, radius=radius, generated_points=generated_points) + + +@dataclass(frozen=True) +class Circle3D: + """ + Contains details of a circle in 3D. + Members + center: tensor shape (3,) + radius: tensor shape () + normal: tensor shape (3,) + generated_points: points around the circle, shape (n_points, 3) + """ + + center: torch.Tensor + radius: torch.Tensor + normal: torch.Tensor + generated_points: torch.Tensor + + +def fit_circle_in_3d( + points, + *, + n_points: int = 0, + angles: Optional[torch.Tensor] = None, + offset: Optional[torch.Tensor] = None, + up: Optional[torch.Tensor] = None, +) -> Circle3D: + """ + Simple best fit circle to 3D points. Uses circle_2d in the + least-squares best fit plane. + + In addition, generates points along the circle. If angles is None (default) + then n_points around the circle equally spaced are given. These begin at the + point closest to the first input point. They continue in the direction which + seems to be match the movement of points. If angles is provided, then n_points + is ignored, and points along the circle at the given angles are returned, + with the starting point and direction as before. + + Further, an offset can be given to add to the generated points; this is + interpreted in a rotated coordinate system where (0, 0, 1) is normal to the + circle, specifically the normal which is approximately in the direction of a + given `up` vector. The remaining rotation is disambiguated in an unspecified + but deterministic way. + + (Note that `generated_points` is affected by the order of the points in + points, but the other outputs are not.) + + Args: + points2d: N x 3 tensor of 3D points + n_points: number of points to generate on the circle + angles: optional angles in radians of points to generate. + offset: optional tensor (3,), a displacement expressed in a "canonical" + coordinate system to add to the generated points. + up: optional tensor (3,), a vector which helps define the + "canonical" coordinate system for interpretting `offset`. + Required if offset is used. + + + Returns: + Circle3D object + """ + centroid = points.mean(0) + r = get_rotation_to_best_fit_xy(points, centroid) + normal = r[:, 2] + rotated_points = (points - centroid) @ r + result_2d = fit_circle_in_2d( + rotated_points[:, :2], n_points=n_points, angles=angles + ) + center_3d = result_2d.center @ r[:, :2].t() + centroid + n_generated_points = result_2d.generated_points.shape[0] + if n_generated_points > 0: + generated_points_in_plane = torch.cat( + [ + result_2d.generated_points, + torch.zeros_like(result_2d.generated_points[:, :1]), + ], + dim=1, + ) + if offset is not None: + if up is None: + raise ValueError("Missing `up` input for interpreting offset") + with torch.no_grad(): + swap = torch.dot(up, normal) < 0 + if swap: + # We need some rotation which takes +z to -z. Here's one. + generated_points_in_plane += offset * offset.new_tensor([1, -1, -1]) + else: + generated_points_in_plane += offset + + generated_points = generated_points_in_plane @ r.t() + centroid + else: + generated_points = points.new_zeros(0, 3) + + return Circle3D( + radius=result_2d.radius, + center=center_3d, + normal=normal, + generated_points=generated_points, + ) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/config.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/config.py new file mode 100644 index 0000000000000000000000000000000000000000..4479fe65562fce952d39f15d520285650977299c --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/config.py @@ -0,0 +1,1210 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import dataclasses +import inspect +import itertools +import sys +import warnings +from collections import Counter, defaultdict +from enum import Enum +from functools import partial +from typing import ( + Any, + Callable, + Dict, + get_args, + get_origin, + List, + Optional, + Tuple, + Type, + TypeVar, + Union, +) + +from omegaconf import DictConfig, OmegaConf, open_dict + + +""" +This functionality allows a configurable system to be determined in a dataclass-type +way. It is a generalization of omegaconf's "structured", in the dataclass case. +Core functionality: + +- Configurable -- A base class used to label a class as being one which uses this + system. Uses class members and __post_init__ like a dataclass. + +- expand_args_fields -- Expands a class like `dataclasses.dataclass`. Runs automatically. + +- get_default_args -- gets an omegaconf.DictConfig for initializing a given class. + +- run_auto_creation -- Initialises nested members. To be called in __post_init__. + + +In addition, a Configurable may contain members whose type is decided at runtime. + +- ReplaceableBase -- As a base instead of Configurable, labels a class to say that + any child class can be used instead. + +- registry -- A global store of named child classes of ReplaceableBase classes. + Used as `@registry.register` decorator on class definition. + + +Additional utility functions: + +- remove_unused_components -- used for simplifying a DictConfig instance. +- get_default_args_field -- default for DictConfig member of another configurable. +- enable_get_default_args -- Allows get_default_args on a function or plain class. + + +1. The simplest usage of this functionality is as follows. First a schema is defined +in dataclass style. + + class A(Configurable): + n: int = 9 + + class B(Configurable): + a: A + + def __post_init__(self): + run_auto_creation(self) + +Then it can be used like + + b_args = get_default_args(B) + b = B(**b_args) + +In this case, get_default_args(B) returns an omegaconf.DictConfig with the right +members {"a_args": {"n": 9}}. It also modifies the definitions of the classes to +something like the following. (The modification itself is done by the function +`expand_args_fields`, which is called inside `get_default_args`.) + + @dataclasses.dataclass + class A: + n: int = 9 + + @dataclasses.dataclass + class B: + a_args: DictConfig = dataclasses.field(default_factory=lambda: DictConfig({"n": 9})) + + def __post_init__(self): + self.a = A(**self.a_args) + +2. Pluggability. Instead of a dataclass-style member being given a concrete class, +it can be given a base class and the implementation will be looked up by name in the +global `registry` in this module. E.g. + + class A(ReplaceableBase): + k: int = 1 + + @registry.register + class A1(A): + m: int = 3 + + @registry.register + class A2(A): + n: str = "2" + + class B(Configurable): + a: A + a_class_type: str = "A2" + b: Optional[A] + b_class_type: Optional[str] = "A2" + + def __post_init__(self): + run_auto_creation(self) + +will expand to + + @dataclasses.dataclass + class A: + k: int = 1 + + @dataclasses.dataclass + class A1(A): + m: int = 3 + + @dataclasses.dataclass + class A2(A): + n: str = "2" + + @dataclasses.dataclass + class B: + a_class_type: str = "A2" + a_A1_args: DictConfig = dataclasses.field( + default_factory=lambda: DictConfig({"k": 1, "m": 3} + ) + a_A2_args: DictConfig = dataclasses.field( + default_factory=lambda: DictConfig({"k": 1, "n": 2} + ) + b_class_type: Optional[str] = "A2" + b_A1_args: DictConfig = dataclasses.field( + default_factory=lambda: DictConfig({"k": 1, "m": 3} + ) + b_A2_args: DictConfig = dataclasses.field( + default_factory=lambda: DictConfig({"k": 1, "n": 2} + ) + + def __post_init__(self): + if self.a_class_type == "A1": + self.a = A1(**self.a_A1_args) + elif self.a_class_type == "A2": + self.a = A2(**self.a_A2_args) + else: + raise ValueError(...) + + if self.b_class_type is None: + self.b = None + elif self.b_class_type == "A1": + self.b = A1(**self.b_A1_args) + elif self.b_class_type == "A2": + self.b = A2(**self.b_A2_args) + else: + raise ValueError(...) + +3. Aside from these classes, the members of these classes should be things +which DictConfig is happy with: e.g. (bool, int, str, None, float) and what +can be built from them with `DictConfig`s and lists of them. + +In addition, you can call `get_default_args` on a function or class to get +the `DictConfig` of its defaulted arguments, assuming those are all things +which `DictConfig` is happy with, so long as you add a call to +`enable_get_default_args` after its definition. If you want to use such a +thing as the default for a member of another configured class, +`get_default_args_field` is a helper. +""" + + +TYPE_SUFFIX: str = "_class_type" +ARGS_SUFFIX: str = "_args" +ENABLED_SUFFIX: str = "_enabled" +CREATE_PREFIX: str = "create_" +IMPL_SUFFIX: str = "_impl" +TWEAK_SUFFIX: str = "_tweak_args" +_DATACLASS_INIT: str = "__dataclass_own_init__" +PRE_EXPAND_NAME: str = "pre_expand" + + +class ReplaceableBase: + """ + Base class for a class (a "replaceable") which is a base class for + dataclass-style implementations. The implementations can be stored + in the registry. They get expanded into dataclasses with expand_args_fields. + This expansion is delayed. + """ + + def __new__(cls, *args, **kwargs): + """ + These classes should be expanded only when needed (because processing + fixes the list of replaceable subclasses of members of the class). It + is safer if users expand the classes explicitly. But if the class gets + instantiated when it hasn't been processed, we expand it here. + """ + obj = super().__new__(cls) + if cls is not ReplaceableBase and not _is_actually_dataclass(cls): + expand_args_fields(cls) + return obj + + +class Configurable: + """ + Base class for dataclass-style classes which are not replaceable. These get + expanded into a dataclass with expand_args_fields. + This expansion is delayed. + """ + + def __new__(cls, *args, **kwargs): + """ + These classes should be expanded only when needed (because processing + fixes the list of replaceable subclasses of members of the class). It + is safer if users expand the classes explicitly. But if the class gets + instantiated when it hasn't been processed, we expand it here. + """ + obj = super().__new__(cls) + if cls is not Configurable and not _is_actually_dataclass(cls): + expand_args_fields(cls) + return obj + + +_X = TypeVar("X", bound=ReplaceableBase) +_Y = TypeVar("Y", bound=Union[ReplaceableBase, Configurable]) + + +class _Registry: + """ + Register from names to classes. In particular, we say that direct subclasses of + ReplaceableBase are "base classes" and we register subclasses of each base class + in a separate namespace. + """ + + def __init__(self) -> None: + self._mapping: Dict[Type[ReplaceableBase], Dict[str, Type[ReplaceableBase]]] = ( + defaultdict(dict) + ) + + def register(self, some_class: Type[_X]) -> Type[_X]: + """ + A class decorator, to register a class in self. + """ + name = some_class.__name__ + self._register(some_class, name=name) + return some_class + + def _register( + self, + some_class: Type[ReplaceableBase], + *, + base_class: Optional[Type[ReplaceableBase]] = None, + name: str, + ) -> None: + """ + Register a new member. + + Args: + cls: the new member + base_class: (optional) what the new member is a type for + name: name for the new member + """ + if base_class is None: + base_class = self._base_class_from_class(some_class) + if base_class is None: + raise ValueError( + f"Cannot register {some_class}. Cannot tell what it is." + ) + self._mapping[base_class][name] = some_class + + def get(self, base_class_wanted: Type[_X], name: str) -> Type[_X]: + """ + Retrieve a class from the registry by name + + Args: + base_class_wanted: parent type of type we are looking for. + It determines the namespace. + This will typically be a direct subclass of ReplaceableBase. + name: what to look for + + Returns: + class type + """ + if self._is_base_class(base_class_wanted): + base_class = base_class_wanted + else: + base_class = self._base_class_from_class(base_class_wanted) + if base_class is None: + raise ValueError( + f"Cannot look up {base_class_wanted}. Cannot tell what it is." + ) + if not isinstance(name, str): + raise ValueError( + f"Cannot look up a {type(name)} in the registry. Got {name}." + ) + result = self._mapping[base_class].get(name) + if result is None: + raise ValueError(f"{name} has not been registered.") + if not issubclass(result, base_class_wanted): + raise ValueError( + f"{name} resolves to {result} which does not subclass {base_class_wanted}" + ) + # pyre-ignore[7] + return result + + def get_all( + self, base_class_wanted: Type[ReplaceableBase] + ) -> List[Type[ReplaceableBase]]: + """ + Retrieve all registered implementations from the registry + + Args: + base_class_wanted: parent type of type we are looking for. + It determines the namespace. + This will typically be a direct subclass of ReplaceableBase. + Returns: + list of class types in alphabetical order of registered name. + """ + if self._is_base_class(base_class_wanted): + source = self._mapping[base_class_wanted] + return [source[key] for key in sorted(source)] + + base_class = self._base_class_from_class(base_class_wanted) + if base_class is None: + raise ValueError( + f"Cannot look up {base_class_wanted}. Cannot tell what it is." + ) + source = self._mapping[base_class] + return [ + source[key] + for key in sorted(source) + if issubclass(source[key], base_class_wanted) + and source[key] is not base_class_wanted + ] + + @staticmethod + def _is_base_class(some_class: Type[ReplaceableBase]) -> bool: + """ + Return whether the given type is a direct subclass of ReplaceableBase + and so gets used as a namespace. + """ + return ReplaceableBase in some_class.__bases__ + + @staticmethod + def _base_class_from_class( + some_class: Type[ReplaceableBase], + ) -> Optional[Type[ReplaceableBase]]: + """ + Find the parent class of some_class which inherits ReplaceableBase, or None + """ + for base in some_class.mro()[-3::-1]: + if base is not ReplaceableBase and issubclass(base, ReplaceableBase): + return base + return None + + +# Global instance of the registry +registry = _Registry() + + +class _ProcessType(Enum): + """ + Type of member which gets rewritten by expand_args_fields. + """ + + CONFIGURABLE = 1 + REPLACEABLE = 2 + OPTIONAL_CONFIGURABLE = 3 + OPTIONAL_REPLACEABLE = 4 + + +def _default_create( + name: str, type_: Type, process_type: _ProcessType +) -> Callable[[Any], None]: + """ + Return the default creation function for a member. This is a function which + could be called in __post_init__ to initialise the member, and will be called + from run_auto_creation. + + Args: + name: name of the member + type_: type of the member (with any Optional removed) + process_type: Shows whether member's declared type inherits ReplaceableBase, + in which case the actual type to be created is decided at + runtime. + + Returns: + Function taking one argument, the object whose member should be + initialized, i.e. self. + """ + impl_name = f"{CREATE_PREFIX}{name}{IMPL_SUFFIX}" + + def inner(self): + expand_args_fields(type_) + impl = getattr(self, impl_name) + args = getattr(self, name + ARGS_SUFFIX) + impl(True, args) + + def inner_optional(self): + expand_args_fields(type_) + impl = getattr(self, impl_name) + enabled = getattr(self, name + ENABLED_SUFFIX) + args = getattr(self, name + ARGS_SUFFIX) + impl(enabled, args) + + def inner_pluggable(self): + type_name = getattr(self, name + TYPE_SUFFIX) + impl = getattr(self, impl_name) + if type_name is None: + args = None + else: + args = getattr(self, f"{name}_{type_name}{ARGS_SUFFIX}", None) + impl(type_name, args) + + if process_type == _ProcessType.OPTIONAL_CONFIGURABLE: + return inner_optional + return inner if process_type == _ProcessType.CONFIGURABLE else inner_pluggable + + +def _default_create_impl( + name: str, type_: Type, process_type: _ProcessType +) -> Callable[[Any, Any, DictConfig], None]: + """ + Return the default internal function for initialising a member. This is a function + which could be called in the create_ function to initialise the member. + + Args: + name: name of the member + type_: type of the member (with any Optional removed) + process_type: Shows whether member's declared type inherits ReplaceableBase, + in which case the actual type to be created is decided at + runtime. + + Returns: + Function taking + - self, the object whose member should be initialized. + - option for what to do. This is + - for pluggables, the type to initialise or None to do nothing + - for non pluggables, a bool indicating whether to initialise. + - the args for initializing the member. + """ + + def create_configurable(self, enabled, args): + if enabled: + expand_args_fields(type_) + setattr(self, name, type_(**args)) + else: + setattr(self, name, None) + + def create_pluggable(self, type_name, args): + if type_name is None: + setattr(self, name, None) + return + + if not isinstance(type_name, str): + raise ValueError( + f"A {type(type_name)} was received as the type of {name}." + + f" Perhaps this is from {name}{TYPE_SUFFIX}?" + ) + chosen_class = registry.get(type_, type_name) + if self._known_implementations.get(type_name, chosen_class) is not chosen_class: + # If this warning is raised, it means that a new definition of + # the chosen class has been registered since our class was processed + # (i.e. expanded). A DictConfig which comes from our get_default_args + # (which might have triggered the processing) will contain the old default + # values for the members of the chosen class. Changes to those defaults which + # were made in the redefinition will not be reflected here. + warnings.warn(f"New implementation of {type_name} is being chosen.") + expand_args_fields(chosen_class) + setattr(self, name, chosen_class(**args)) + + if process_type in (_ProcessType.CONFIGURABLE, _ProcessType.OPTIONAL_CONFIGURABLE): + return create_configurable + return create_pluggable + + +def run_auto_creation(self: Any) -> None: + """ + Run all the functions named in self._creation_functions. + """ + for create_function in self._creation_functions: + getattr(self, create_function)() + + +def _is_configurable_class(C) -> bool: + return isinstance(C, type) and issubclass(C, (Configurable, ReplaceableBase)) + + +def get_default_args(C, *, _do_not_process: Tuple[type, ...] = ()) -> DictConfig: + """ + Get the DictConfig corresponding to the defaults in a dataclass or + configurable. Normal use is to provide a dataclass can be provided as C. + If enable_get_default_args has been called on a function or plain class, + then that function or class can be provided as C. + + If C is a subclass of Configurable or ReplaceableBase, we make sure + it has been processed with expand_args_fields. + + Args: + C: the class or function to be processed + _do_not_process: (internal use) When this function is called from + expand_args_fields, we specify any class currently being + processed, to make sure we don't try to process a class + while it is already being processed. + + Returns: + new DictConfig object, which is typed. + """ + if C is None: + return DictConfig({}) + + if _is_configurable_class(C): + if C in _do_not_process: + raise ValueError( + f"Internal recursion error. Need processed {C}," + f" but cannot get it. _do_not_process={_do_not_process}" + ) + # This is safe to run multiple times. It will return + # straight away if C has already been processed. + expand_args_fields(C, _do_not_process=_do_not_process) + + if dataclasses.is_dataclass(C): + # Note that if get_default_args_field is used somewhere in C, + # this call is recursive. No special care is needed, + # because in practice get_default_args_field is used for + # separate types than the outer type. + + try: + out: DictConfig = OmegaConf.structured(C) + except Exception: + print(f"### OmegaConf.structured({C}) failed ###") + # We don't use `raise From` here, because that gets the original + # exception hidden by the OC_CAUSE logic in the case where we are + # called by hydra. + raise + exclude = getattr(C, "_processed_members", ()) + with open_dict(out): + for field in exclude: + out.pop(field, None) + return out + + if _is_configurable_class(C): + raise ValueError(f"Failed to process {C}") + + if not inspect.isfunction(C) and not inspect.isclass(C): + raise ValueError(f"Unexpected {C}") + + dataclass_name = _dataclass_name_for_function(C) + dataclass = getattr(sys.modules[C.__module__], dataclass_name, None) + if dataclass is None: + raise ValueError( + f"Cannot get args for {C}. Was enable_get_default_args forgotten?" + ) + + try: + out: DictConfig = OmegaConf.structured(dataclass) + except Exception: + print(f"### OmegaConf.structured failed for {C.__name__} ###") + raise + return out + + +def _dataclass_name_for_function(C: Any) -> str: + """ + Returns the name of the dataclass which enable_get_default_args(C) + creates. + """ + name = f"_{C.__name__}_default_args_" + return name + + +def _field_annotations_for_default_args( + C: Any, +) -> List[Tuple[str, Any, dataclasses.Field]]: + """ + If C is a function or a plain class with an __init__ function, + return the fields which `enable_get_default_args(C)` will need + to make a dataclass with. + + Args: + C: a function, or a class with an __init__ function. Must + have types for all its defaulted args. + + Returns: + a list of fields for a dataclass. + """ + + field_annotations = [] + for pname, defval in _params_iter(C): + default = defval.default + if default == inspect.Parameter.empty: + # we do not have a default value for the parameter + continue + + if defval.annotation == inspect._empty: + raise ValueError( + "All arguments of the input to enable_get_default_args have to" + f" be typed. Argument '{pname}' does not have a type annotation." + ) + + _, annotation = _resolve_optional(defval.annotation) + + if isinstance(default, set): # force OmegaConf to convert it to ListConfig + default = tuple(default) + + if isinstance(default, (list, dict)): + # OmegaConf will convert to [Dict|List]Config, so it is safe to reuse the value + field_ = dataclasses.field(default_factory=lambda default=default: default) + elif not _is_immutable_type(annotation, default): + continue + else: + # we can use a simple default argument for dataclass.field + field_ = dataclasses.field(default=default) + field_annotations.append((pname, defval.annotation, field_)) + + return field_annotations + + +def enable_get_default_args(C: Any, *, overwrite: bool = True) -> None: + """ + If C is a function or a plain class with an __init__ function, + and you want get_default_args(C) to work, then add + `enable_get_default_args(C)` straight after the definition of C. + This makes a dataclass corresponding to the default arguments of C + and stores it in the same module as C. + + Args: + C: a function, or a class with an __init__ function. Must + have types for all its defaulted args. + overwrite: whether to allow calling this a second time on + the same function. + """ + if not inspect.isfunction(C) and not inspect.isclass(C): + raise ValueError(f"Unexpected {C}") + + field_annotations = _field_annotations_for_default_args(C) + + name = _dataclass_name_for_function(C) + module = sys.modules[C.__module__] + if hasattr(module, name): + if overwrite: + warnings.warn(f"Overwriting {name} in {C.__module__}.") + else: + raise ValueError(f"Cannot overwrite {name} in {C.__module__}.") + dc = dataclasses.make_dataclass(name, field_annotations) + dc.__module__ = C.__module__ + setattr(module, name, dc) + + +def _params_iter(C): + """Returns dict of keyword args of a class or function C.""" + if inspect.isclass(C): + return itertools.islice( # exclude `self` + inspect.signature(C.__init__).parameters.items(), 1, None + ) + + return inspect.signature(C).parameters.items() + + +def _is_immutable_type(type_: Type, val: Any) -> bool: + if val is None: + return True + + PRIMITIVE_TYPES = (int, float, bool, str, bytes, tuple) + # sometimes type can be too relaxed (e.g. Any), so we also check values + if isinstance(val, PRIMITIVE_TYPES): + return True + + return type_ in PRIMITIVE_TYPES or ( + inspect.isclass(type_) and issubclass(type_, Enum) + ) + + +# copied from OmegaConf +def _resolve_optional(type_: Any) -> Tuple[bool, Any]: + """Check whether `type_` is equivalent to `typing.Optional[T]` for some T.""" + if get_origin(type_) is Union: + args = get_args(type_) + if len(args) == 2 and args[1] == type(None): # noqa E721 + return True, args[0] + if type_ is Any: + return True, Any + + return False, type_ + + +def _is_actually_dataclass(some_class) -> bool: + # Return whether the class some_class has been processed with + # the dataclass annotation. This is more specific than + # dataclasses.is_dataclass which returns True on anything + # deriving from a dataclass. + + # Checking for __init__ would also work for our purpose. + return "__dataclass_fields__" in some_class.__dict__ + + +def expand_args_fields( + some_class: Type[_Y], *, _do_not_process: Tuple[type, ...] = () +) -> Type[_Y]: + """ + This expands a class which inherits Configurable or ReplaceableBase classes, + including dataclass processing. some_class is modified in place by this function. + If expand_args_fields(some_class) has already been called, subsequent calls do + nothing and return some_class unmodified. + For classes of type ReplaceableBase, you can add some_class to the registry before + or after calling this function. But potential inner classes need to be registered + before this function is run on the outer class. + + The transformations this function makes, before the concluding + dataclasses.dataclass, are as follows. If X is a base class with registered + subclasses Y and Z, replace a class member + + x: X + + and optionally + + x_class_type: str = "Y" + def create_x(self):... + + with + + x_Y_args: dict = dataclasses.field(default_factory=lambda: get_default_args(Y)) + x_Z_args: dict = dataclasses.field(default_factory=lambda: get_default_args(Z)) + def create_x(self): + args = self.getattr(f"x_{self.x_class_type}_args") + self.create_x_impl(self.x_class_type, args) + def create_x_impl(self, x_type, args): + x_type = registry.get(X, x_type) + expand_args_fields(x_type) + self.x = x_type(**args) + x_class_type: str = "UNDEFAULTED" + + without adding the optional attributes if they are already there. + + Similarly, replace + + x: Optional[X] + + and optionally + + x_class_type: Optional[str] = "Y" + def create_x(self):... + + with + + x_Y_args: dict = dataclasses.field(default_factory=lambda: get_default_args(Y)) + x_Z_args: dict = dataclasses.field(default_factory=lambda: get_default_args(Z)) + def create_x(self): + if self.x_class_type is None: + args = None + else: + args = self.getattr(f"x_{self.x_class_type}_args", None) + self.create_x_impl(self.x_class_type, args) + def create_x_impl(self, x_class_type, args): + if x_class_type is None: + self.x = None + return + + x_type = registry.get(X, x_class_type) + expand_args_fields(x_type) + assert args is not None + self.x = x_type(**args) + x_class_type: Optional[str] = "UNDEFAULTED" + + without adding the optional attributes if they are already there. + + Similarly, if X is a subclass of Configurable, + + x: X + + and optionally + + def create_x(self):... + + will be replaced with + + x_args: dict = dataclasses.field(default_factory=lambda: get_default_args(X)) + def create_x(self): + self.create_x_impl(True, self.x_args) + + def create_x_impl(self, enabled, args): + if enabled: + expand_args_fields(X) + self.x = X(**args) + else: + self.x = None + + Similarly, replace, + + x: Optional[X] + x_enabled: bool = ... + + and optionally + + def create_x(self):... + + with + + x_args: dict = dataclasses.field(default_factory=lambda: get_default_args(X)) + x_enabled: bool = ... + def create_x(self): + self.create_x_impl(self.x_enabled, self.x_args) + + def create_x_impl(self, enabled, args): + if enabled: + expand_args_fields(X) + self.x = X(**args) + else: + self.x = None + + + Also adds the following class members, unannotated so that dataclass + ignores them. + - _creation_functions: Tuple[str, ...] of all the create_ functions, + including those from base classes (not the create_x_impl ones). + - _known_implementations: Dict[str, Type] containing the classes which + have been found from the registry. + (used only to raise a warning if it one has been overwritten) + - _processed_members: a Dict[str, Any] of all the members which have been + transformed, with values giving the types they were declared to have. + (E.g. {"x": X} or {"x": Optional[X]} in the cases above.) + + In addition, if the class has a member function + + @classmethod + def x_tweak_args(cls, member_type: Type, args: DictConfig) -> None + + then the default_factory of x_args will also have a call to x_tweak_args(X, x_args) and + the default_factory of x_Y_args will also have a call to x_tweak_args(Y, x_Y_args). + + In addition, if the class inherits torch.nn.Module, the generated __init__ will + call torch.nn.Module's __init__ before doing anything else. + + Before any transformation of the class, if the class has a classmethod called + `pre_expand`, it will be called with no arguments. + + Note that although the *_args members are intended to have type DictConfig, they + are actually internally annotated as dicts. OmegaConf is happy to see a DictConfig + in place of a dict, but not vice-versa. Allowing dict lets a class user specify + x_args as an explicit dict without getting an incomprehensible error. + + Args: + some_class: the class to be processed + _do_not_process: Internal use for get_default_args: Because get_default_args calls + and is called by this function, we let it specify any class currently + being processed, to make sure we don't try to process a class while + it is already being processed. + + + Returns: + some_class itself, which has been modified in place. This + allows this function to be used as a class decorator. + """ + if _is_actually_dataclass(some_class): + return some_class + + if hasattr(some_class, PRE_EXPAND_NAME): + getattr(some_class, PRE_EXPAND_NAME)() + + # The functions this class's run_auto_creation will run. + creation_functions: List[str] = [] + # The classes which this type knows about from the registry + # We could use a weakref.WeakValueDictionary here which would mean + # that we don't warn if the class we should have expected is elsewhere + # unused. + known_implementations: Dict[str, Type] = {} + # Names of members which have been processed. + processed_members: Dict[str, Any] = {} + + # For all bases except ReplaceableBase and Configurable and object, + # we need to process them before our own processing. This is + # because dataclasses expect to inherit dataclasses and not unprocessed + # dataclasses. + for base in some_class.mro()[-3:0:-1]: + if base is ReplaceableBase: + continue + if base is Configurable: + continue + if not issubclass(base, (Configurable, ReplaceableBase)): + continue + expand_args_fields(base, _do_not_process=_do_not_process) + if "_creation_functions" in base.__dict__: + creation_functions.extend(base._creation_functions) + if "_known_implementations" in base.__dict__: + known_implementations.update(base._known_implementations) + if "_processed_members" in base.__dict__: + processed_members.update(base._processed_members) + + to_process: List[Tuple[str, Type, _ProcessType]] = [] + if "__annotations__" in some_class.__dict__: + for name, type_ in some_class.__annotations__.items(): + underlying_and_process_type = _get_type_to_process(type_) + if underlying_and_process_type is None: + continue + underlying_type, process_type = underlying_and_process_type + to_process.append((name, underlying_type, process_type)) + + for name, underlying_type, process_type in to_process: + processed_members[name] = some_class.__annotations__[name] + _process_member( + name=name, + type_=underlying_type, + process_type=process_type, + some_class=some_class, + creation_functions=creation_functions, + _do_not_process=_do_not_process, + known_implementations=known_implementations, + ) + + for key, count in Counter(creation_functions).items(): + if count > 1: + warnings.warn(f"Clash with {key} in a base class.") + some_class._creation_functions = tuple(creation_functions) + some_class._processed_members = processed_members + some_class._known_implementations = known_implementations + + dataclasses.dataclass(eq=False)(some_class) + _fixup_class_init(some_class) + return some_class + + +def _fixup_class_init(some_class) -> None: + """ + In-place modification of the some_class class which happens + after dataclass processing. + + If the dataclass some_class inherits torch.nn.Module, then + makes torch.nn.Module's __init__ be called before anything else + on instantiation of some_class. + This is a bit like attr's __pre_init__. + """ + + assert _is_actually_dataclass(some_class) + try: + import torch + except ModuleNotFoundError: + return + + if not issubclass(some_class, torch.nn.Module): + return + + def init(self, *args, **kwargs) -> None: + torch.nn.Module.__init__(self) + getattr(self, _DATACLASS_INIT)(*args, **kwargs) + + assert _DATACLASS_INIT not in some_class.__dict__ + + setattr(some_class, _DATACLASS_INIT, some_class.__init__) + some_class.__init__ = init + + +def get_default_args_field( + C, + *, + _do_not_process: Tuple[type, ...] = (), + _hook: Optional[Callable[[DictConfig], None]] = None, +): + """ + Get a dataclass field which defaults to get_default_args(...) + + Args: + C: As for get_default_args. + _do_not_process: As for get_default_args + _hook: Function called on the result before returning. + + Returns: + function to return new DictConfig object + """ + + def create(): + args = get_default_args(C, _do_not_process=_do_not_process) + if _hook is not None: + with open_dict(args): + _hook(args) + return args + + return dataclasses.field(default_factory=create) + + +def _get_default_args_field_from_registry( + *, + base_class_wanted: Type[_X], + name: str, + _do_not_process: Tuple[type, ...] = (), + _hook: Optional[Callable[[DictConfig], None]] = None, +): + """ + Get a dataclass field which defaults to + get_default_args(registry.get(base_class_wanted, name)). + + This is used internally in place of get_default_args_field in + order that default values are updated if a class is redefined. + + Args: + base_class_wanted: As for registry.get. + name: As for registry.get. + _do_not_process: As for get_default_args + _hook: Function called on the result before returning. + + Returns: + function to return new DictConfig object + """ + + def create(): + C = registry.get(base_class_wanted=base_class_wanted, name=name) + args = get_default_args(C, _do_not_process=_do_not_process) + if _hook is not None: + with open_dict(args): + _hook(args) + return args + + return dataclasses.field(default_factory=create) + + +def _get_type_to_process(type_) -> Optional[Tuple[Type, _ProcessType]]: + """ + If a member is annotated as `type_`, and that should expanded in + expand_args_fields, return how it should be expanded. + """ + if get_origin(type_) == Union: + # We look for Optional[X] which is a Union of X with None. + args = get_args(type_) + if len(args) != 2 or all(a is not type(None) for a in args): # noqa: E721 + return + underlying = args[0] if args[1] is type(None) else args[1] # noqa: E721 + if ( + isinstance(underlying, type) + and issubclass(underlying, ReplaceableBase) + and ReplaceableBase in underlying.__bases__ + ): + return underlying, _ProcessType.OPTIONAL_REPLACEABLE + + if isinstance(underlying, type) and issubclass(underlying, Configurable): + return underlying, _ProcessType.OPTIONAL_CONFIGURABLE + + if not isinstance(type_, type): + # e.g. any other Union or Tuple. Or ClassVar. + return + + if issubclass(type_, ReplaceableBase) and ReplaceableBase in type_.__bases__: + return type_, _ProcessType.REPLACEABLE + + if issubclass(type_, Configurable): + return type_, _ProcessType.CONFIGURABLE + + +def _process_member( + *, + name: str, + type_: Type, + process_type: _ProcessType, + some_class: Type, + creation_functions: List[str], + _do_not_process: Tuple[type, ...], + known_implementations: Dict[str, Type], +) -> None: + """ + Make the modification (of expand_args_fields) to some_class for a single member. + + Args: + name: member name + type_: member type (with Optional removed if needed) + process_type: whether member has dynamic type + some_class: (MODIFIED IN PLACE) the class being processed + creation_functions: (MODIFIED IN PLACE) the names of the create functions + _do_not_process: as for expand_args_fields. + known_implementations: (MODIFIED IN PLACE) known types from the registry + """ + # Because we are adding defaultable members, make + # sure they go at the end of __annotations__ in case + # there are non-defaulted standard class members. + del some_class.__annotations__[name] + hook = getattr(some_class, name + TWEAK_SUFFIX, None) + + if process_type in (_ProcessType.REPLACEABLE, _ProcessType.OPTIONAL_REPLACEABLE): + type_name = name + TYPE_SUFFIX + if type_name not in some_class.__annotations__: + if process_type == _ProcessType.OPTIONAL_REPLACEABLE: + some_class.__annotations__[type_name] = Optional[str] + else: + some_class.__annotations__[type_name] = str + setattr(some_class, type_name, "UNDEFAULTED") + + for derived_type in registry.get_all(type_): + if derived_type in _do_not_process: + continue + if issubclass(derived_type, some_class): + # When derived_type is some_class we have a simple + # recursion to avoid. When it's a strict subclass the + # situation is even worse. + continue + known_implementations[derived_type.__name__] = derived_type + args_name = f"{name}_{derived_type.__name__}{ARGS_SUFFIX}" + if args_name in some_class.__annotations__: + raise ValueError( + f"Cannot generate {args_name} because it is already present." + ) + some_class.__annotations__[args_name] = dict + if hook is not None: + hook_closed = partial(hook, derived_type) + else: + hook_closed = None + setattr( + some_class, + args_name, + _get_default_args_field_from_registry( + base_class_wanted=type_, + name=derived_type.__name__, + _do_not_process=_do_not_process + (some_class,), + _hook=hook_closed, + ), + ) + else: + args_name = name + ARGS_SUFFIX + if args_name in some_class.__annotations__: + raise ValueError( + f"Cannot generate {args_name} because it is already present." + ) + if issubclass(type_, some_class) or type_ in _do_not_process: + raise ValueError(f"Cannot process {type_} inside {some_class}") + + some_class.__annotations__[args_name] = dict + if hook is not None: + hook_closed = partial(hook, type_) + else: + hook_closed = None + setattr( + some_class, + args_name, + get_default_args_field( + type_, + _do_not_process=_do_not_process + (some_class,), + _hook=hook_closed, + ), + ) + if process_type == _ProcessType.OPTIONAL_CONFIGURABLE: + enabled_name = name + ENABLED_SUFFIX + if enabled_name not in some_class.__annotations__: + raise ValueError( + f"{name} is an Optional[{type_.__name__}] member " + f"but there is no corresponding member {enabled_name}." + ) + + creation_function_name = f"{CREATE_PREFIX}{name}" + if not hasattr(some_class, creation_function_name): + setattr( + some_class, + creation_function_name, + _default_create(name, type_, process_type), + ) + creation_functions.append(creation_function_name) + + creation_function_impl_name = f"{CREATE_PREFIX}{name}{IMPL_SUFFIX}" + if not hasattr(some_class, creation_function_impl_name): + setattr( + some_class, + creation_function_impl_name, + _default_create_impl(name, type_, process_type), + ) + + +def remove_unused_components(dict_: DictConfig) -> None: + """ + Assuming dict_ represents the state of a configurable, + modify it to remove all the portions corresponding to + pluggable parts which are not in use. + For example, if renderer_class_type is SignedDistanceFunctionRenderer, + the renderer_MultiPassEmissionAbsorptionRenderer_args will be + removed. Also, if chocolate_enabled is False, then chocolate_args will + be removed. + + Args: + dict_: (MODIFIED IN PLACE) a DictConfig instance + """ + keys = [key for key in dict_ if isinstance(key, str)] + suffix_length = len(TYPE_SUFFIX) + replaceables = [key[:-suffix_length] for key in keys if key.endswith(TYPE_SUFFIX)] + args_keys = [key for key in keys if key.endswith(ARGS_SUFFIX)] + for replaceable in replaceables: + selected_type = dict_[replaceable + TYPE_SUFFIX] + if selected_type is None: + expect = "" + else: + expect = replaceable + "_" + selected_type + ARGS_SUFFIX + with open_dict(dict_): + for key in args_keys: + if key.startswith(replaceable + "_") and key != expect: + del dict_[key] + + suffix_length = len(ENABLED_SUFFIX) + enableables = [key[:-suffix_length] for key in keys if key.endswith(ENABLED_SUFFIX)] + for enableable in enableables: + enabled = dict_[enableable + ENABLED_SUFFIX] + if not enabled: + with open_dict(dict_): + dict_.pop(enableable + ARGS_SUFFIX, None) + + for key in dict_: + if isinstance(dict_.get(key), DictConfig): + remove_unused_components(dict_[key]) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/depth_cleanup.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/depth_cleanup.py new file mode 100644 index 0000000000000000000000000000000000000000..89d48d37c37c9660171506eaf378a31c6f058082 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/depth_cleanup.py @@ -0,0 +1,115 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import torch +import torch.nn.functional as Fu +from pytorch3d.ops import wmean +from pytorch3d.renderer.cameras import CamerasBase +from pytorch3d.structures import Pointclouds + + +def cleanup_eval_depth( + point_cloud: Pointclouds, + camera: CamerasBase, + depth: torch.Tensor, + mask: torch.Tensor, + sigma: float = 0.01, + image=None, +): + + ba, _, H, W = depth.shape + + pcl = point_cloud.points_padded() + n_pts = point_cloud.num_points_per_cloud() + pcl_mask = ( + torch.arange(pcl.shape[1], dtype=torch.int64, device=pcl.device)[None] + < n_pts[:, None] + ).type_as(pcl) + + pcl_proj = camera.transform_points(pcl, eps=1e-2)[..., :-1] + pcl_depth = camera.get_world_to_view_transform().transform_points(pcl)[..., -1] + + depth_and_idx = torch.cat( + ( + depth, + torch.arange(H * W).view(1, 1, H, W).expand(ba, 1, H, W).type_as(depth), + ), + dim=1, + ) + + depth_and_idx_sampled = Fu.grid_sample( + depth_and_idx, -pcl_proj[:, None], mode="nearest" + )[:, :, 0].view(ba, 2, -1) + + depth_sampled, idx_sampled = depth_and_idx_sampled.split([1, 1], dim=1) + df = (depth_sampled[:, 0] - pcl_depth).abs() + + # the threshold is a sigma-multiple of the standard deviation of the depth + mu = wmean(depth.view(ba, -1, 1), mask.view(ba, -1)).view(ba, 1) + std = ( + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and `int`. + wmean((depth.view(ba, -1) - mu).view(ba, -1, 1) ** 2, mask.view(ba, -1)) + .clamp(1e-4) + .sqrt() + .view(ba, -1) + ) + good_df_thr = std * sigma + good_depth = (df <= good_df_thr).float() * pcl_mask + + # perc_kept = good_depth.sum(dim=1) / pcl_mask.sum(dim=1).clamp(1) + # print(f'Kept {100.0 * perc_kept.mean():1.3f} % points') + + good_depth_raster = torch.zeros_like(depth).view(ba, -1) + good_depth_raster.scatter_add_(1, torch.round(idx_sampled[:, 0]).long(), good_depth) + + good_depth_mask = (good_depth_raster.view(ba, 1, H, W) > 0).float() + + # if float(torch.rand(1)) > 0.95: + # depth_ok = depth * good_depth_mask + + # # visualize + # visdom_env = 'depth_cleanup_dbg' + # from visdom import Visdom + # # from tools.vis_utils import make_depth_image + # from pytorch3d.vis.plotly_vis import plot_scene + # viz = Visdom() + + # show_pcls = { + # 'pointclouds': point_cloud, + # } + # for d, nm in zip( + # (depth, depth_ok), + # ('pointclouds_unproj', 'pointclouds_unproj_ok'), + # ): + # pointclouds_unproj = get_rgbd_point_cloud( + # camera, image, d, + # ) + # if int(pointclouds_unproj.num_points_per_cloud()) > 0: + # show_pcls[nm] = pointclouds_unproj + + # scene_dict = {'1': { + # **show_pcls, + # 'cameras': camera, + # }} + # scene = plot_scene( + # scene_dict, + # pointcloud_max_points=5000, + # pointcloud_marker_size=1.5, + # camera_scale=1.0, + # ) + # viz.plotlyplot(scene, env=visdom_env, win='scene') + + # # depth_image_ok = make_depth_image(depths_ok, masks) + # # viz.images(depth_image_ok, env=visdom_env, win='depth_ok') + # # depth_image = make_depth_image(depths, masks) + # # viz.images(depth_image, env=visdom_env, win='depth') + # # # viz.images(rgb_rendered, env=visdom_env, win='images_render') + # # viz.images(images, env=visdom_env, win='images') + # import pdb; pdb.set_trace() + + return good_depth_mask diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/eval_video_trajectory.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/eval_video_trajectory.py new file mode 100644 index 0000000000000000000000000000000000000000..f46fe292f8e45c1d40ef157f444941ee23761e16 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/eval_video_trajectory.py @@ -0,0 +1,270 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import logging +import math +from typing import Optional, Tuple + +import torch +from pytorch3d.implicitron.tools import utils +from pytorch3d.implicitron.tools.circle_fitting import fit_circle_in_3d +from pytorch3d.renderer import look_at_view_transform, PerspectiveCameras +from pytorch3d.transforms import Scale + + +logger = logging.getLogger(__name__) + + +def generate_eval_video_cameras( + train_cameras, + n_eval_cams: int = 100, + trajectory_type: str = "figure_eight", + trajectory_scale: float = 0.2, + scene_center: Tuple[float, float, float] = (0.0, 0.0, 0.0), + up: Tuple[float, float, float] = (0.0, 0.0, 1.0), + focal_length: Optional[torch.Tensor] = None, + principal_point: Optional[torch.Tensor] = None, + time: Optional[torch.Tensor] = None, + infer_up_as_plane_normal: bool = True, + traj_offset: Optional[Tuple[float, float, float]] = None, + traj_offset_canonical: Optional[Tuple[float, float, float]] = None, + remove_outliers_rate: float = 0.0, +) -> PerspectiveCameras: + """ + Generate a camera trajectory rendering a scene from multiple viewpoints. + + Args: + train_cameras: The set of cameras from the training dataset object. + n_eval_cams: Number of cameras in the trajectory. + trajectory_type: The type of the camera trajectory. Can be one of: + circular_lsq_fit: Camera centers follow a trajectory obtained + by fitting a 3D circle to train_cameras centers. + All cameras are looking towards scene_center. + figure_eight: Figure-of-8 trajectory around the center of the + central camera of the training dataset. + trefoil_knot: Same as 'figure_eight', but the trajectory has a shape + of a trefoil knot (https://en.wikipedia.org/wiki/Trefoil_knot). + figure_eight_knot: Same as 'figure_eight', but the trajectory has a shape + of a figure-eight knot + (https://en.wikipedia.org/wiki/Figure-eight_knot_(mathematics)). + trajectory_scale: The extent of the trajectory. + scene_center: The center of the scene in world coordinates which all + the cameras from the generated trajectory look at. + up: The "circular_lsq_fit" vector of the scene (=the normal of the scene floor). + Active for the `trajectory_type="circular"`. + focal_length: The focal length of the output cameras. If `None`, an average + focal length of the train_cameras is used. + principal_point: The principal point of the output cameras. If `None`, an average + principal point of all train_cameras is used. + time: Defines the total length of the generated camera trajectory. All possible + trajectories (set with the `trajectory_type` argument) are periodic with + the period of `time=2pi`. + E.g. setting `trajectory_type=circular_lsq_fit` and `time=4pi`, will generate + a trajectory of camera poses rotating the total of 720 deg around the object. + infer_up_as_plane_normal: Infer the camera `up` vector automatically as the normal + of the plane fit to the optical centers of `train_cameras`. + traj_offset: 3D offset vector added to each point of the trajectory. + traj_offset_canonical: 3D offset vector expressed in the local coordinates of + the estimated trajectory which is added to each point of the trajectory. + remove_outliers_rate: the number between 0 and 1; if > 0, + some outlier train_cameras will be removed from trajectory estimation; + the filtering is based on camera center coordinates; top and + bottom `remove_outliers_rate` cameras on each dimension are removed. + Returns: + Batch of camera instances which can be used as the test dataset + """ + if remove_outliers_rate > 0.0: + train_cameras = _remove_outlier_cameras(train_cameras, remove_outliers_rate) + + if trajectory_type in ("figure_eight", "trefoil_knot", "figure_eight_knot"): + cam_centers = train_cameras.get_camera_center() + # get the nearest camera center to the mean of centers + mean_camera_idx = ( + ((cam_centers - cam_centers.mean(dim=0)[None]) ** 2) + .sum(dim=1) + .min(dim=0) + .indices + ) + # generate the knot trajectory in canonical coords + if time is None: + time = torch.linspace(0, 2 * math.pi, n_eval_cams + 1)[:n_eval_cams] + else: + assert time.numel() == n_eval_cams + if trajectory_type == "trefoil_knot": + traj = _trefoil_knot(time) + elif trajectory_type == "figure_eight_knot": + traj = _figure_eight_knot(time) + elif trajectory_type == "figure_eight": + traj = _figure_eight(time) + else: + raise ValueError(f"bad trajectory type: {trajectory_type}") + traj[:, 2] -= traj[:, 2].max() + + # transform the canonical knot to the coord frame of the mean camera + mean_camera = PerspectiveCameras( + **{ + k: getattr(train_cameras, k)[[int(mean_camera_idx)]] + for k in ("focal_length", "principal_point", "R", "T") + } + ) + traj_trans = Scale(cam_centers.std(dim=0).mean() * trajectory_scale).compose( + mean_camera.get_world_to_view_transform().inverse() + ) + + if traj_offset_canonical is not None: + traj_trans = traj_trans.translate( + torch.FloatTensor(traj_offset_canonical)[None].to(traj) + ) + + traj = traj_trans.transform_points(traj) + + plane_normal = _fit_plane(cam_centers)[:, 0] + if infer_up_as_plane_normal: + up = _disambiguate_normal(plane_normal, up) + + elif trajectory_type == "circular_lsq_fit": + ### fit plane to the camera centers + + # get the center of the plane as the median of the camera centers + cam_centers = train_cameras.get_camera_center() + + if time is not None: + angle = time + else: + angle = torch.linspace(0, 2.0 * math.pi, n_eval_cams).to(cam_centers) + + fit = fit_circle_in_3d( + cam_centers, + angles=angle, + offset=( + angle.new_tensor(traj_offset_canonical) + if traj_offset_canonical is not None + else None + ), + up=angle.new_tensor(up), + ) + traj = fit.generated_points + + # scalethe trajectory + _t_mu = traj.mean(dim=0, keepdim=True) + traj = (traj - _t_mu) * trajectory_scale + _t_mu + + plane_normal = fit.normal + + if infer_up_as_plane_normal: + up = _disambiguate_normal(plane_normal, up) + + else: + raise ValueError(f"Uknown trajectory_type {trajectory_type}.") + + if traj_offset is not None: + traj = traj + torch.FloatTensor(traj_offset)[None].to(traj) + + # point all cameras towards the center of the scene + R, T = look_at_view_transform( + eye=traj, + at=(scene_center,), # (1, 3) + up=(up,), # (1, 3) + device=traj.device, + ) + + # get the average focal length and principal point + if focal_length is None: + focal_length = train_cameras.focal_length.mean(dim=0).repeat(n_eval_cams, 1) + if principal_point is None: + principal_point = train_cameras.principal_point.mean(dim=0).repeat( + n_eval_cams, 1 + ) + + test_cameras = PerspectiveCameras( + focal_length=focal_length, + principal_point=principal_point, + R=R, + T=T, + device=focal_length.device, + ) + + # _visdom_plot_scene( + # train_cameras, + # test_cameras, + # ) + + return test_cameras + + +def _remove_outlier_cameras( + cameras: PerspectiveCameras, outlier_rate: float +) -> PerspectiveCameras: + keep_indices = utils.get_inlier_indicators( + cameras.get_camera_center(), dim=0, outlier_rate=outlier_rate + ) + # pyre-fixme[6]: For 1st param expected `Union[List[int], int, BoolTensor, + # LongTensor]` but got `Tensor`. + clean_cameras = cameras[keep_indices] + logger.info( + "Filtered outlier cameras when estimating the trajectory: " + f"{len(cameras)} → {len(clean_cameras)}" + ) + # pyre-fixme[7]: Expected `PerspectiveCameras` but got `CamerasBase`. + return clean_cameras + + +def _disambiguate_normal(normal, up): + up_t = torch.tensor(up).to(normal) + flip = (up_t * normal).sum().sign() + up = normal * flip + up = up.tolist() + return up + + +def _fit_plane(x): + x = x - x.mean(dim=0)[None] + cov = (x.t() @ x) / x.shape[0] + _, e_vec = torch.linalg.eigh(cov) + return e_vec + + +def _visdom_plot_scene( + train_cameras, + test_cameras, +) -> None: + from pytorch3d.vis.plotly_vis import plot_scene + + p = plot_scene( + { + "scene": { + "train_cams": train_cameras, + "test_cams": test_cameras, + } + } + ) + from visdom import Visdom + + viz = Visdom() + viz.plotlyplot(p, env="cam_traj_dbg", win="cam_trajs") + + +def _figure_eight_knot(t: torch.Tensor, z_scale: float = 0.5): + x = (2 + (2 * t).cos()) * (3 * t).cos() + y = (2 + (2 * t).cos()) * (3 * t).sin() + z = (4 * t).sin() * z_scale + return torch.stack((x, y, z), dim=-1) + + +def _trefoil_knot(t: torch.Tensor, z_scale: float = 0.5): + x = t.sin() + 2 * (2 * t).sin() + y = t.cos() - 2 * (2 * t).cos() + z = -(3 * t).sin() * z_scale + return torch.stack((x, y, z), dim=-1) + + +def _figure_eight(t: torch.Tensor, z_scale: float = 0.5): + x = t.cos() + y = (2 * t).sin() / 2 + z = t.sin() * z_scale + return torch.stack((x, y, z), dim=-1) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/image_utils.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/image_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7e8b3b04dbb9c1c9fd59d88cbdef0cf39623bbf2 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/image_utils.py @@ -0,0 +1,57 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +from typing import Sequence, Union + +import torch + + +def mask_background( + image_rgb: torch.Tensor, + mask_fg: torch.Tensor, + dim_color: int = 1, + bg_color: Union[torch.Tensor, Sequence, str, float] = 0.0, +) -> torch.Tensor: + """ + Mask the background input image tensor `image_rgb` with `bg_color`. + The background regions are obtained from the binary foreground segmentation + mask `mask_fg`. + """ + tgt_view = [1, 1, 1, 1] + tgt_view[dim_color] = 3 + # obtain the background color tensor + if isinstance(bg_color, torch.Tensor): + bg_color_t = bg_color.view(1, 3, 1, 1).clone().to(image_rgb) + elif isinstance(bg_color, (float, tuple, list)): + if isinstance(bg_color, float): + bg_color = [bg_color] * 3 + bg_color_t = torch.tensor( + bg_color, device=image_rgb.device, dtype=image_rgb.dtype + ).view(*tgt_view) + elif isinstance(bg_color, str): + if bg_color == "white": + bg_color_t = image_rgb.new_ones(tgt_view) + elif bg_color == "black": + bg_color_t = image_rgb.new_zeros(tgt_view) + else: + raise ValueError(_invalid_color_error_msg(bg_color)) + else: + raise ValueError(_invalid_color_error_msg(bg_color)) + # cast to the image_rgb's type + mask_fg = mask_fg.type_as(image_rgb) + # mask the bg + image_masked = mask_fg * image_rgb + (1 - mask_fg) * bg_color_t + return image_masked + + +def _invalid_color_error_msg(bg_color) -> str: + return ( + f"Invalid bg_color={bg_color}. Plese set bg_color to a 3-element" + + " tensor. or a string (white | black), or a float." + ) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/metric_utils.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/metric_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d4debeef422e489e7ce58a048eb2174cc6201c2b --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/metric_utils.py @@ -0,0 +1,237 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import math +from typing import Optional, Tuple + +import torch +from torch.nn import functional as F + + +def eval_depth( + pred: torch.Tensor, + gt: torch.Tensor, + crop: int = 1, + mask: Optional[torch.Tensor] = None, + get_best_scale: bool = True, + mask_thr: float = 0.5, + best_scale_clamp_thr: float = 1e-4, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Evaluate the depth error between the prediction `pred` and the ground + truth `gt`. + + Args: + pred: A tensor of shape (N, 1, H, W) denoting the predicted depth maps. + gt: A tensor of shape (N, 1, H, W) denoting the ground truth depth maps. + crop: The number of pixels to crop from the border. + mask: A mask denoting the valid regions of the gt depth. + get_best_scale: If `True`, estimates a scaling factor of the predicted depth + that yields the best mean squared error between `pred` and `gt`. + This is typically enabled for cases where predicted reconstructions + are inherently defined up to an arbitrary scaling factor. + mask_thr: A constant used to threshold the `mask` to specify the valid + regions. + best_scale_clamp_thr: The threshold for clamping the divisor in best + scale estimation. + + Returns: + mse_depth: Mean squared error between `pred` and `gt`. + abs_depth: Mean absolute difference between `pred` and `gt`. + """ + + # chuck out the border + if crop > 0: + gt = gt[:, :, crop:-crop, crop:-crop] + pred = pred[:, :, crop:-crop, crop:-crop] + + if mask is not None: + # mult gt by mask + if crop > 0: + mask = mask[:, :, crop:-crop, crop:-crop] + gt = gt * (mask > mask_thr).float() + + dmask = (gt > 0.0).float() + dmask_mass = torch.clamp(dmask.sum((1, 2, 3)), 1e-4) + + if get_best_scale: + # mult preds by a scalar "scale_best" + # s.t. we get best possible mse error + scale_best = estimate_depth_scale_factor(pred, gt, dmask, best_scale_clamp_thr) + pred = pred * scale_best[:, None, None, None] + + df = gt - pred + + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and `int`. + mse_depth = (dmask * (df**2)).sum((1, 2, 3)) / dmask_mass + abs_depth = (dmask * df.abs()).sum((1, 2, 3)) / dmask_mass + + return mse_depth, abs_depth + + +def estimate_depth_scale_factor(pred, gt, mask, clamp_thr): + xy = pred * gt * mask + xx = pred * pred * mask + scale_best = xy.mean((1, 2, 3)) / torch.clamp(xx.mean((1, 2, 3)), clamp_thr) + return scale_best + + +def calc_psnr( + x: torch.Tensor, + y: torch.Tensor, + mask: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """ + Calculates the Peak-signal-to-noise ratio between tensors `x` and `y`. + """ + mse = calc_mse(x, y, mask=mask) + psnr = torch.log10(mse.clamp(1e-10)) * (-10.0) + return psnr + + +def calc_mse( + x: torch.Tensor, + y: torch.Tensor, + mask: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """ + Calculates the mean square error between tensors `x` and `y`. + """ + if mask is None: + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and `int`. + return torch.mean((x - y) ** 2) + else: + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and `int`. + return (((x - y) ** 2) * mask).sum() / mask.expand_as(x).sum().clamp(1e-5) + + +def calc_bce( + pred: torch.Tensor, + gt: torch.Tensor, + equal_w: bool = True, + pred_eps: float = 0.01, + mask: Optional[torch.Tensor] = None, + lerp_bound: Optional[float] = None, +) -> torch.Tensor: + """ + Calculates the binary cross entropy. + """ + if pred_eps > 0.0: + # up/low bound the predictions + pred = torch.clamp(pred, pred_eps, 1.0 - pred_eps) + + if mask is None: + mask = torch.ones_like(gt) + + if equal_w: + mask_fg = (gt > 0.5).float() * mask + mask_bg = (1 - mask_fg) * mask + weight = mask_fg / mask_fg.sum().clamp(1.0) + mask_bg / mask_bg.sum().clamp(1.0) + # weight sum should be at this point ~2 + # pyre-fixme[58]: `/` is not supported for operand types `int` and `Tensor`. + weight = weight * (weight.numel() / weight.sum().clamp(1.0)) + else: + weight = torch.ones_like(gt) * mask + + if lerp_bound is not None: + return binary_cross_entropy_lerp(pred, gt, weight, lerp_bound) + else: + return F.binary_cross_entropy(pred, gt, reduction="mean", weight=weight) + + +def binary_cross_entropy_lerp( + pred: torch.Tensor, + gt: torch.Tensor, + weight: torch.Tensor, + lerp_bound: float, +): + """ + Binary cross entropy which avoids exploding gradients by linearly + extrapolating the log function for log(1-pred) mad log(pred) whenever + pred or 1-pred is smaller than lerp_bound. + """ + loss = log_lerp(1 - pred, lerp_bound) * (1 - gt) + log_lerp(pred, lerp_bound) * gt + loss_reduced = -(loss * weight).sum() / weight.sum().clamp(1e-4) + return loss_reduced + + +def log_lerp(x: torch.Tensor, b: float): + """ + Linearly extrapolated log for x < b. + """ + assert b > 0 + return torch.where(x >= b, x.log(), math.log(b) + (x - b) / b) + + +def rgb_l1( + pred: torch.Tensor, target: torch.Tensor, mask: Optional[torch.Tensor] = None +) -> torch.Tensor: + """ + Calculates the mean absolute error between the predicted colors `pred` + and ground truth colors `target`. + """ + if mask is None: + mask = torch.ones_like(pred[:, :1]) + return ((pred - target).abs() * mask).sum(dim=(1, 2, 3)) / mask.sum( + dim=(1, 2, 3) + ).clamp(1) + + +def huber(dfsq: torch.Tensor, scaling: float = 0.03) -> torch.Tensor: + """ + Calculates the huber function of the input squared error `dfsq`. + The function smoothly transitions from a region with unit gradient + to a hyperbolic function at `dfsq=scaling`. + """ + loss = (safe_sqrt(1 + dfsq / (scaling * scaling), eps=1e-4) - 1) * scaling + return loss + + +def neg_iou_loss( + predict: torch.Tensor, + target: torch.Tensor, + mask: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """ + This is a great loss because it emphasizes on the active + regions of the predict and targets + """ + return 1.0 - iou(predict, target, mask=mask) + + +def safe_sqrt(A: torch.Tensor, eps: float = 1e-4) -> torch.Tensor: + """ + performs safe differentiable sqrt + """ + return (torch.clamp(A, float(0)) + eps).sqrt() + + +def iou( + predict: torch.Tensor, + target: torch.Tensor, + mask: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """ + This is a great loss because it emphasizes on the active + regions of the predict and targets + """ + dims = tuple(range(predict.dim())[1:]) + if mask is not None: + predict = predict * mask + target = target * mask + intersect = (predict * target).sum(dims) + union = (predict + target - predict * target).sum(dims) + 1e-4 + return (intersect / union).sum() / intersect.numel() + + +def beta_prior(pred: torch.Tensor, cap: float = 0.1) -> torch.Tensor: + if cap <= 0.0: + raise ValueError("capping should be positive to avoid unbound loss") + + min_value = math.log(cap) + math.log(cap + 1.0) + return (torch.log(pred + cap) + torch.log(1.0 - pred + cap)).mean() - min_value diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/model_io.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/model_io.py new file mode 100644 index 0000000000000000000000000000000000000000..c6bc499d698632fdb2424141769ef4b16eb7168a --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/model_io.py @@ -0,0 +1,175 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import glob +import logging +import os +import shutil +import tempfile +from typing import Optional + +import torch + + +logger = logging.getLogger(__name__) + + +def load_stats(flstats): + from pytorch3d.implicitron.tools.stats import Stats + + if not os.path.isfile(flstats): + return None + + return Stats.load(flstats) + + +def get_model_path(fl) -> str: + fl = os.path.splitext(fl)[0] + flmodel = "%s.pth" % fl + return flmodel + + +def get_optimizer_path(fl) -> str: + fl = os.path.splitext(fl)[0] + flopt = "%s_opt.pth" % fl + return flopt + + +def get_stats_path(fl, eval_results: bool = False) -> str: + fl = os.path.splitext(fl)[0] + if eval_results: + for postfix in ("_2", ""): + flstats = os.path.join(os.path.dirname(fl), f"stats_test{postfix}.jgz") + if os.path.isfile(flstats): + break + else: + flstats = "%s_stats.jgz" % fl + # pyre-fixme[61]: `flstats` is undefined, or not always defined. + return flstats + + +def safe_save_model(model, stats, fl, optimizer=None, cfg=None) -> None: + """ + This functions stores model files safely so that no model files exist on the + file system in case the saving procedure gets interrupted. + + This is done first by saving the model files to a temporary directory followed + by (atomic) moves to the target location. Note, that this can still result + in a corrupt set of model files in case interruption happens while performing + the moves. It is however quite improbable that a crash would occur right at + this time. + """ + logger.info(f"saving model files safely to {fl}") + # first store everything to a tmpdir + with tempfile.TemporaryDirectory() as tmpdir: + tmpfl = os.path.join(tmpdir, os.path.split(fl)[-1]) + stored_tmp_fls = save_model(model, stats, tmpfl, optimizer=optimizer, cfg=cfg) + tgt_fls = [ + ( + os.path.join(os.path.split(fl)[0], os.path.split(tmpfl)[-1]) + if (tmpfl is not None) + else None + ) + for tmpfl in stored_tmp_fls + ] + # then move from the tmpdir to the right location + for tmpfl, tgt_fl in zip(stored_tmp_fls, tgt_fls): + if tgt_fl is None: + continue + shutil.move(tmpfl, tgt_fl) + + +def save_model(model, stats, fl, optimizer=None, cfg=None): + flstats = get_stats_path(fl) + flmodel = get_model_path(fl) + logger.info("saving model to %s" % flmodel) + torch.save(model.state_dict(), flmodel) + flopt = None + if optimizer is not None: + flopt = get_optimizer_path(fl) + logger.info("saving optimizer to %s" % flopt) + torch.save(optimizer.state_dict(), flopt) + logger.info("saving model stats to %s" % flstats) + stats.save(flstats) + + return flstats, flmodel, flopt + + +def save_stats(stats, fl, cfg=None): + flstats = get_stats_path(fl) + logger.info("saving model stats to %s" % flstats) + stats.save(flstats) + return flstats + + +def load_model(fl, map_location: Optional[dict]): + flstats = get_stats_path(fl) + flmodel = get_model_path(fl) + flopt = get_optimizer_path(fl) + model_state_dict = torch.load(flmodel, map_location=map_location) + stats = load_stats(flstats) + if os.path.isfile(flopt): + optimizer = torch.load(flopt, map_location=map_location) + else: + optimizer = None + + return model_state_dict, stats, optimizer + + +def parse_epoch_from_model_path(model_path) -> int: + return int( + os.path.split(model_path)[-1].replace(".pth", "").replace("model_epoch_", "") + ) + + +def get_checkpoint(exp_dir, epoch): + fl = os.path.join(exp_dir, "model_epoch_%08d.pth" % epoch) + return fl + + +def find_last_checkpoint( + exp_dir, any_path: bool = False, all_checkpoints: bool = False +): + if any_path: + exts = [".pth", "_stats.jgz", "_opt.pth"] + else: + exts = [".pth"] + + for ext in exts: + fls = sorted( + glob.glob( + os.path.join(glob.escape(exp_dir), "model_epoch_" + "[0-9]" * 8 + ext) + ) + ) + if len(fls) > 0: + break + # pyre-fixme[61]: `fls` is undefined, or not always defined. + if len(fls) == 0: + fl = None + else: + if all_checkpoints: + # pyre-fixme[61]: `fls` is undefined, or not always defined. + fl = [f[0 : -len(ext)] + ".pth" for f in fls] + else: + # pyre-fixme[61]: `ext` is undefined, or not always defined. + fl = fls[-1][0 : -len(ext)] + ".pth" + + return fl + + +def purge_epoch(exp_dir, epoch) -> None: + model_path = get_checkpoint(exp_dir, epoch) + + for file_path in [ + model_path, + get_optimizer_path(model_path), + get_stats_path(model_path), + ]: + if os.path.isfile(file_path): + logger.info("deleting %s" % file_path) + os.remove(file_path) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/point_cloud_utils.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/point_cloud_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..feee053ea67b282ce64affe50ec2f596d34d04db --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/point_cloud_utils.py @@ -0,0 +1,197 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +from typing import cast, Optional, Tuple + +import torch +import torch.nn.functional as Fu +from pytorch3d.renderer import ( + AlphaCompositor, + NDCMultinomialRaysampler, + PointsRasterizationSettings, + PointsRasterizer, + ray_bundle_to_ray_points, +) +from pytorch3d.renderer.cameras import CamerasBase +from pytorch3d.structures import Pointclouds + + +def get_rgbd_point_cloud( + camera: CamerasBase, + image_rgb: torch.Tensor, + depth_map: torch.Tensor, + mask: Optional[torch.Tensor] = None, + mask_thr: float = 0.5, + *, + euclidean: bool = False, +) -> Pointclouds: + """ + Given a batch of images, depths, masks and cameras, generate a single colored + point cloud by unprojecting depth maps and coloring with the source + pixel colors. + + Arguments: + camera: Batch of N cameras + image_rgb: Batch of N images of shape (N, C, H, W). + For RGB images C=3. + depth_map: Batch of N depth maps of shape (N, 1, H', W'). + Only positive values here are used to generate points. + If euclidean=False (default) this contains perpendicular distances + from each point to the camera plane (z-values). + If euclidean=True, this contains distances from each point to + the camera center. + mask: If provided, batch of N masks of the same shape as depth_map. + If provided, values in depth_map are ignored if the corresponding + element of mask is smaller than mask_thr. + mask_thr: used in interpreting mask + euclidean: used in interpreting depth_map. + + Returns: + Pointclouds object containing one point cloud. + """ + imh, imw = depth_map.shape[2:] + + # convert the depth maps to point clouds using the grid ray sampler + pts_3d = ray_bundle_to_ray_points( + NDCMultinomialRaysampler( + image_width=imw, + image_height=imh, + n_pts_per_ray=1, + min_depth=1.0, + max_depth=1.0, + unit_directions=euclidean, + )(camera)._replace(lengths=depth_map[:, 0, ..., None]) + ) + + pts_mask = depth_map > 0.0 + if mask is not None: + pts_mask *= mask > mask_thr + pts_mask = pts_mask.reshape(-1) + + pts_3d = pts_3d.reshape(-1, 3)[pts_mask] + + pts_colors = torch.nn.functional.interpolate( + image_rgb, + size=[imh, imw], + mode="bilinear", + align_corners=False, + ) + pts_colors = pts_colors.permute(0, 2, 3, 1).reshape(-1, image_rgb.shape[1])[ + pts_mask + ] + + return Pointclouds(points=pts_3d[None], features=pts_colors[None]) + + +def render_point_cloud_pytorch3d( + camera, + point_cloud, + render_size: Tuple[int, int], + point_radius: float = 0.03, + topk: int = 10, + eps: float = 1e-2, + bg_color=None, + bin_size: Optional[int] = None, + **kwargs, +): + + # feature dimension + featdim = point_cloud.features_packed().shape[-1] + + # move to the camera coordinates; using identity cameras in the renderer + point_cloud = _transform_points(camera, point_cloud, eps, **kwargs) + camera_trivial = camera.clone() + camera_trivial.R[:] = torch.eye(3) + camera_trivial.T *= 0.0 + + bin_size = ( + bin_size + if bin_size is not None + else (64 if int(max(render_size)) > 1024 else None) + ) + rasterizer = PointsRasterizer( + cameras=camera_trivial, + raster_settings=PointsRasterizationSettings( + image_size=render_size, + radius=point_radius, + points_per_pixel=topk, + bin_size=bin_size, + ), + ) + + fragments = rasterizer(point_cloud, **kwargs) + + # Construct weights based on the distance of a point to the true point. + # However, this could be done differently: e.g. predicted as opposed + # to a function of the weights. + r = rasterizer.raster_settings.radius + + # set up the blending weights + dists2 = fragments.dists + weights = 1 - dists2 / (r * r) + ok = cast(torch.BoolTensor, (fragments.idx >= 0)).float() + + weights = weights * ok + + fragments_prm = fragments.idx.long().permute(0, 3, 1, 2) + weights_prm = weights.permute(0, 3, 1, 2) + images = AlphaCompositor()( + fragments_prm, + weights_prm, + point_cloud.features_packed().permute(1, 0), + background_color=bg_color if bg_color is not None else [0.0] * featdim, + **kwargs, + ) + + # get the depths ... + # weighted_fs[b,c,i,j] = sum_k cum_alpha_k * features[c,pointsidx[b,k,i,j]] + # cum_alpha_k = alphas[b,k,i,j] * prod_l=0..k-1 (1 - alphas[b,l,i,j]) + cumprod = torch.cumprod(1 - weights, dim=-1) + cumprod = torch.cat((torch.ones_like(cumprod[..., :1]), cumprod[..., :-1]), dim=-1) + depths = (weights * cumprod * fragments.zbuf).sum(dim=-1) + # add the rendering mask + # pyre-fixme[6]: For 1st param expected `Tensor` but got `float`. + render_mask = -torch.prod(1.0 - weights, dim=-1) + 1.0 + + # cat depths and render mask + rendered_blob = torch.cat((images, depths[:, None], render_mask[:, None]), dim=1) + + # reshape back + rendered_blob = Fu.interpolate( + rendered_blob, + size=tuple(render_size), + mode="bilinear", + align_corners=False, + ) + + data_rendered, depth_rendered, render_mask = rendered_blob.split( + [rendered_blob.shape[1] - 2, 1, 1], + dim=1, + ) + + return data_rendered, render_mask, depth_rendered + + +def _signed_clamp(x, eps): + sign = x.sign() + (x == 0.0).type_as(x) + x_clamp = sign * torch.clamp(x.abs(), eps) + return x_clamp + + +def _transform_points(cameras, point_clouds, eps, **kwargs): + pts_world = point_clouds.points_padded() + pts_view = cameras.get_world_to_view_transform(**kwargs).transform_points( + pts_world, eps=eps + ) + # it is crucial to actually clamp the points as well ... + pts_view = torch.cat( + (pts_view[..., :-1], _signed_clamp(pts_view[..., -1:], eps)), dim=-1 + ) + point_clouds = point_clouds.update_padded(pts_view) + return point_clouds diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/rasterize_mc.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/rasterize_mc.py new file mode 100644 index 0000000000000000000000000000000000000000..9615987eeda913011f06c4a21a2d1e7f24395998 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/rasterize_mc.py @@ -0,0 +1,147 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import math +from typing import Optional, Tuple + +import pytorch3d + +import torch +from pytorch3d.ops import packed_to_padded +from pytorch3d.renderer import PerspectiveCameras +from pytorch3d.structures import Pointclouds + +from .point_cloud_utils import render_point_cloud_pytorch3d + + +@torch.no_grad() +def rasterize_sparse_ray_bundle( + ray_bundle: "pytorch3d.implicitron.models.renderer.base.ImplicitronRayBundle", + features: torch.Tensor, + image_size_hw: Tuple[int, int], + depth: torch.Tensor, + masks: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Rasterizes sparse features corresponding to the coordinates defined by + the rays in the bundle. + + Args: + ray_bundle: ray bundle object with B x ... x 2 pixel coordinates, + it can be packed. + features: B x ... x C tensor containing per-point rendered features. + image_size_hw: Tuple[image_height, image_width] containing + the size of rasterized image. + depth: B x ... x 1 tensor containing per-point rendered depth. + masks: B x ... x 1 tensor containing the alpha mask of the + rendered features. + + Returns: + - image_render: B x C x H x W tensor of rasterized features + - depths_render: B x 1 x H x W tensor of rasterized depth maps + - masks_render: B x 1 x H x W tensor of opacities after splatting + """ + # Flatten the features and xy locations. + features_depth_ras = torch.cat( + (features.flatten(1, -2), depth.flatten(1, -2)), dim=-1 + ) + xys = ray_bundle.xys + masks_ras = None + if ray_bundle.is_packed(): + camera_counts = ray_bundle.camera_counts + assert camera_counts is not None + xys, first_idxs, _ = ray_bundle.get_padded_xys() + masks_ras = ( + torch.arange(xys.shape[1], device=xys.device)[:, None] + < camera_counts[:, None, None] + ) + + max_size = torch.max(camera_counts).item() + features_depth_ras = packed_to_padded( + features_depth_ras[:, 0], first_idxs, max_size + ) + if masks is not None: + padded_mask = packed_to_padded(masks.flatten(1, -1), first_idxs, max_size) + masks_ras = padded_mask * masks_ras + + xys_ras = xys.flatten(1, -2) + + if masks_ras is None: + assert not ray_bundle.is_packed() + masks_ras = masks.flatten(1, -2) if masks is not None else None + + if min(*image_size_hw) <= 0: + raise ValueError( + "Need to specify a positive output_size_hw for bundle rasterisation." + ) + + # Estimate the rasterization point radius so that we approximately fill + # the whole image given the number of rasterized points. + pt_radius = 2.0 / math.sqrt(xys.shape[1]) + + # Rasterize the samples. + features_depth_render, masks_render = rasterize_mc_samples( + xys_ras, + features_depth_ras, + image_size_hw, + radius=pt_radius, + masks=masks_ras, + ) + images_render = features_depth_render[:, :-1] + depths_render = features_depth_render[:, -1:] + return images_render, depths_render, masks_render + + +def rasterize_mc_samples( + xys: torch.Tensor, + feats: torch.Tensor, + image_size_hw: Tuple[int, int], + radius: float = 0.03, + topk: int = 5, + masks: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Rasterizes Monte-Carlo sampled features back onto the image. + + Specifically, the code uses the PyTorch3D point rasterizer to render + a z-flat point cloud composed of the xy MC locations and their features. + + Args: + xys: B x N x 2 2D point locations in PyTorch3D NDC convention + feats: B x N x dim tensor containing per-point rendered features. + image_size_hw: Tuple[image_height, image_width] containing + the size of rasterized image. + radius: Rasterization point radius. + topk: The maximum z-buffer size for the PyTorch3D point cloud rasterizer. + masks: B x N x 1 tensor containing the alpha mask of the + rendered features. + """ + + if masks is None: + masks = torch.ones_like(xys[..., :1]) + + feats = torch.cat((feats, masks), dim=-1) + pointclouds = Pointclouds( + points=torch.cat([xys, torch.ones_like(xys[..., :1])], dim=-1), + features=feats, + ) + + data_rendered, render_mask, _ = render_point_cloud_pytorch3d( + PerspectiveCameras(device=feats.device), + pointclouds, + render_size=image_size_hw, + point_radius=radius, + topk=topk, + ) + + data_rendered, masks_pt = data_rendered.split( + [data_rendered.shape[1] - 1, 1], dim=1 + ) + render_mask = masks_pt * render_mask + + return data_rendered, render_mask diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/stats.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/stats.py new file mode 100644 index 0000000000000000000000000000000000000000..0625f3e38d2070db6cac8f57d74a5475c3a132e5 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/stats.py @@ -0,0 +1,513 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import gzip +import json +import logging +import time +import warnings +from collections.abc import Iterable +from itertools import cycle + +import matplotlib +import matplotlib.pyplot as plt +import numpy as np +from matplotlib import colors as mcolors +from pytorch3d.implicitron.tools.vis_utils import get_visdom_connection + +logger = logging.getLogger(__name__) + + +class AverageMeter: + """Computes and stores the average and current value""" + + def __init__(self): + self.history = [] + self.reset() + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1, epoch=0): + + # make sure the history is of the same len as epoch + while len(self.history) <= epoch: + self.history.append([]) + + self.history[epoch].append(val / n) + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + def get_epoch_averages(self, epoch=-1): + if len(self.history) == 0: # no stats here + return None + elif epoch == -1: + return [ + (float(np.array(x).mean()) if len(x) > 0 else float("NaN")) + for x in self.history + ] + else: + return float(np.array(self.history[epoch]).mean()) + + def get_all_values(self): + all_vals = [np.array(x) for x in self.history] + all_vals = np.concatenate(all_vals) + return all_vals + + def get_epoch(self): + return len(self.history) + + @staticmethod + def from_json_str(json_str): + self = AverageMeter() + self.__dict__.update(json.loads(json_str)) + return self + + +class Stats: + # TODO: update this with context manager + """ + stats logging object useful for gathering statistics of training a deep net in pytorch + Example:: + + # init stats structure that logs statistics 'objective' and 'top1e' + stats = Stats( ('objective','top1e') ) + network = init_net() # init a pytorch module (=nueral network) + dataloader = init_dataloader() # init a dataloader + for epoch in range(10): + # start of epoch -> call new_epoch + stats.new_epoch() + + # iterate over batches + for batch in dataloader: + + output = network(batch) # run and save into a dict of output variables + + # stats.update() automatically parses the 'objective' and 'top1e' from + # the "output" dict and stores this into the db + stats.update(output) + # prints the metric averages over given epoch + std_out = stats.get_status_string() + logger.info(str_out) + # stores the training plots into '/tmp/epoch_stats.pdf' + # and plots into a visdom server running at localhost (if running) + stats.plot_stats(plot_file='/tmp/epoch_stats.pdf') + + """ + + def __init__( + self, + log_vars, + epoch=-1, + visdom_env="main", + do_plot=True, + plot_file=None, + visdom_server="http://localhost", + visdom_port=8097, + ): + + self.log_vars = log_vars + self.visdom_env = visdom_env + self.visdom_server = visdom_server + self.visdom_port = visdom_port + self.plot_file = plot_file + self.do_plot = do_plot + self.hard_reset(epoch=epoch) + self._t_last_update = None + + @staticmethod + def from_json_str(json_str): + self = Stats([]) + # load the global state + self.__dict__.update(json.loads(json_str)) + # recover the AverageMeters + for stat_set in self.stats: + self.stats[stat_set] = { + log_var: AverageMeter.from_json_str(log_vals_json_str) + for log_var, log_vals_json_str in self.stats[stat_set].items() + } + return self + + @staticmethod + def load(flpath, postfix=".jgz"): + flpath = _get_postfixed_filename(flpath, postfix) + with gzip.open(flpath, "r") as fin: + data = json.loads(fin.read().decode("utf-8")) + return Stats.from_json_str(data) + + def save(self, flpath, postfix=".jgz"): + flpath = _get_postfixed_filename(flpath, postfix) + # store into a gzipped-json + with gzip.open(flpath, "w") as fout: + fout.write(json.dumps(self, cls=StatsJSONEncoder).encode("utf-8")) + + # some sugar to be used with "with stats:" at the beginning of the epoch + def __enter__(self): + if self.do_plot and self.epoch >= 0: + self.plot_stats(self.visdom_env) + self.new_epoch() + + def __exit__(self, type, value, traceback): + iserr = type is not None and issubclass(type, Exception) + iserr = iserr or (type is KeyboardInterrupt) + if iserr: + logger.error("error inside 'with' block") + return + if self.do_plot: + self.plot_stats(self.visdom_env) + + def reset(self): # to be called after each epoch + stat_sets = list(self.stats.keys()) + logger.debug(f"stats: epoch {self.epoch} - reset") + self.it = {k: -1 for k in stat_sets} + for stat_set in stat_sets: + for stat in self.stats[stat_set]: + self.stats[stat_set][stat].reset() + + def hard_reset(self, epoch=-1): # to be called during object __init__ + self.epoch = epoch + logger.debug(f"stats: epoch {self.epoch} - hard reset") + self.stats = {} + + # reset + self.reset() + + def new_epoch(self): + logger.debug(f"stats: new epoch {(self.epoch + 1)}") + self.epoch += 1 + self.reset() # zero the stats + increase epoch counter + + def gather_value(self, val): + if isinstance(val, (float, int)): + val = float(val) + else: + val = val.data.cpu().numpy() + val = float(val.sum()) + return val + + def add_log_vars(self, added_log_vars): + for add_log_var in added_log_vars: + if add_log_var not in self.stats: + logger.debug(f"Adding {add_log_var}") + self.log_vars.append(add_log_var) + + def update(self, preds, time_start=None, freeze_iter=False, stat_set="train"): + + if self.epoch == -1: # uninitialized + logger.warning( + "epoch==-1 means uninitialized stats structure -> new_epoch() called" + ) + self.new_epoch() + + if stat_set not in self.stats: + self.stats[stat_set] = {} + self.it[stat_set] = -1 + + if not freeze_iter: + self.it[stat_set] += 1 + + epoch = self.epoch + + for stat in self.log_vars: + + if stat not in self.stats[stat_set]: + self.stats[stat_set][stat] = AverageMeter() + + if stat == "sec/it": # compute speed + if time_start is None: + time_per_it = 0.0 + else: + now = time.time() + time_per_it = now - (self._t_last_update or time_start) + self._t_last_update = now + val = time_per_it + else: + if stat in preds: + try: + val = self.gather_value(preds[stat]) + except KeyError: + raise ValueError( + "could not extract prediction %s\ + from the prediction dictionary" + % stat + ) from None + else: + val = None + + if val is not None: + self.stats[stat_set][stat].update(val, epoch=epoch, n=1) + + def get_epoch_averages(self, epoch=None): + + stat_sets = list(self.stats.keys()) + + if epoch is None: + epoch = self.epoch + if epoch == -1: + epoch = list(range(self.epoch)) + + outvals = {} + for stat_set in stat_sets: + outvals[stat_set] = { + "epoch": epoch, + "it": self.it[stat_set], + "epoch_max": self.epoch, + } + + for stat in self.stats[stat_set].keys(): + if self.stats[stat_set][stat].count == 0: + continue + if isinstance(epoch, Iterable): + avgs = self.stats[stat_set][stat].get_epoch_averages() + avgs = [avgs[e] for e in epoch] + else: + avgs = self.stats[stat_set][stat].get_epoch_averages(epoch=epoch) + outvals[stat_set][stat] = avgs + + return outvals + + def print( + self, + max_it=None, + stat_set="train", + vars_print=None, + get_str=False, + skip_nan=False, + stat_format=lambda s: s.replace("loss_", "").replace("prev_stage_", "ps_"), + ): + """ + stats.print() is deprecated. Please use get_status_string() instead. + example: + std_out = stats.get_status_string() + logger.info(str_out) + """ + + epoch = self.epoch + stats = self.stats + + str_out = "" + + it = self.it[stat_set] + stat_str = "" + stats_print = sorted(stats[stat_set].keys()) + for stat in stats_print: + if stats[stat_set][stat].count == 0: + continue + if skip_nan and not np.isfinite(stats[stat_set][stat].avg): + continue + stat_str += " {0:.12}: {1:1.3f} |".format( + stat_format(stat), stats[stat_set][stat].avg + ) + + head_str = "[%s] | epoch %3d | it %5d" % (stat_set, epoch, it) + if max_it: + head_str += "/ %d" % max_it + + str_out = "%s | %s" % (head_str, stat_str) + + if get_str: + return str_out + else: + warnings.warn( + "get_str=False is deprecated." + "Please enable this flag to get receive the output string.", + DeprecationWarning, + ) + print(str_out) + + def get_status_string( + self, + max_it=None, + stat_set="train", + vars_print=None, + skip_nan=False, + stat_format=lambda s: s.replace("loss_", "").replace("prev_stage_", "ps_"), + ): + return self.print( + max_it=max_it, + stat_set=stat_set, + vars_print=vars_print, + get_str=True, + skip_nan=skip_nan, + stat_format=stat_format, + ) + + def plot_stats( + self, visdom_env=None, plot_file=None, visdom_server=None, visdom_port=None + ): + + # use the cached visdom env if none supplied + if visdom_env is None: + visdom_env = self.visdom_env + if visdom_server is None: + visdom_server = self.visdom_server + if visdom_port is None: + visdom_port = self.visdom_port + if plot_file is None: + plot_file = self.plot_file + + stat_sets = list(self.stats.keys()) + + logger.debug( + f"printing charts to visdom env '{visdom_env}' ({visdom_server}:{visdom_port})" + ) + + novisdom = False + + viz = get_visdom_connection(server=visdom_server, port=visdom_port) + if viz is None or not viz.check_connection(): + logger.info("no visdom server! -> skipping visdom plots") + novisdom = True + + lines = [] + + # plot metrics + if not novisdom: + viz.close(env=visdom_env, win=None) + + for stat in self.log_vars: + vals = [] + stat_sets_now = [] + for stat_set in stat_sets: + val = self.stats[stat_set][stat].get_epoch_averages() + if val is None: + continue + else: + val = np.array(val).reshape(-1) + stat_sets_now.append(stat_set) + vals.append(val) + + if len(vals) == 0: + continue + + lines.append((stat_sets_now, stat, vals)) + + if not novisdom: + for tmodes, stat, vals in lines: + title = "%s" % stat + opts = {"title": title, "legend": list(tmodes)} + for i, (tmode, val) in enumerate(zip(tmodes, vals)): + update = "append" if i > 0 else None + valid = np.where(np.isfinite(val))[0] + if len(valid) == 0: + continue + x = np.arange(len(val)) + viz.line( + Y=val[valid], + X=x[valid], + env=visdom_env, + opts=opts, + win=f"stat_plot_{title}", + name=tmode, + update=update, + ) + + if plot_file: + logger.info(f"plotting stats to {plot_file}") + ncol = 3 + nrow = int(np.ceil(float(len(lines)) / ncol)) + matplotlib.rcParams.update({"font.size": 5}) + color = cycle(plt.cm.tab10(np.linspace(0, 1, 10))) + fig = plt.figure(1) + plt.clf() + for idx, (tmodes, stat, vals) in enumerate(lines): + c = next(color) + plt.subplot(nrow, ncol, idx + 1) + plt.gca() + for vali, vals_ in enumerate(vals): + c_ = c * (1.0 - float(vali) * 0.3) + valid = np.where(np.isfinite(vals_))[0] + if len(valid) == 0: + continue + x = np.arange(len(vals_)) + plt.plot(x[valid], vals_[valid], c=c_, linewidth=1) + plt.ylabel(stat) + plt.xlabel("epoch") + plt.gca().yaxis.label.set_color(c[0:3] * 0.75) + plt.legend(tmodes) + gcolor = np.array(mcolors.to_rgba("lightgray")) + grid_params = {"visible": True, "color": gcolor} + plt.grid(**grid_params, which="major", linestyle="-", linewidth=0.4) + plt.grid(**grid_params, which="minor", linestyle="--", linewidth=0.2) + plt.minorticks_on() + + plt.tight_layout() + plt.show() + try: + fig.savefig(plot_file) + except PermissionError: + warnings.warn("Cant dump stats due to insufficient permissions!") + + def synchronize_logged_vars(self, log_vars, default_val=float("NaN")): + + stat_sets = list(self.stats.keys()) + + # remove the additional log_vars + for stat_set in stat_sets: + for stat in self.stats[stat_set].keys(): + if stat not in log_vars: + logger.warning(f"additional stat {stat_set}:{stat} -> removing") + + self.stats[stat_set] = { + stat: v for stat, v in self.stats[stat_set].items() if stat in log_vars + } + + self.log_vars = log_vars # !!! + + for stat_set in stat_sets: + for stat in log_vars: + if stat not in self.stats[stat_set]: + logger.info( + "missing stat %s:%s -> filling with default values (%1.2f)" + % (stat_set, stat, default_val) + ) + elif len(self.stats[stat_set][stat].history) != self.epoch + 1: + h = self.stats[stat_set][stat].history + if len(h) == 0: # just never updated stat ... skip + continue + else: + logger.info( + "incomplete stat %s:%s -> reseting with default values (%1.2f)" + % (stat_set, stat, default_val) + ) + else: + continue + + self.stats[stat_set][stat] = AverageMeter() + self.stats[stat_set][stat].reset() + + lastep = self.epoch + 1 + for ep in range(lastep): + self.stats[stat_set][stat].update(default_val, n=1, epoch=ep) + epoch_generated = self.stats[stat_set][stat].get_epoch() + assert ( + epoch_generated == self.epoch + 1 + ), "bad epoch of synchronized log_var! %d vs %d" % ( + self.epoch + 1, + epoch_generated, + ) + + +class StatsJSONEncoder(json.JSONEncoder): + def default(self, o): + if isinstance(o, (AverageMeter, Stats)): + enc = self.encode(o.__dict__) + return enc + else: + raise TypeError( + f"Object of type {o.__class__.__name__} " f"is not JSON serializable" + ) + + +def _get_postfixed_filename(fl, postfix): + return fl if fl.endswith(postfix) else fl + postfix diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/utils.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b556d1eba47489c2c1dfc0e3af56b570bf9a80b2 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/utils.py @@ -0,0 +1,207 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +import collections +import dataclasses +import time +from contextlib import contextmanager +from typing import Any, Callable, Dict, Iterable, Iterator + +import torch + + +@contextmanager +def evaluating(net: torch.nn.Module): + """Temporarily switch to evaluation mode.""" + istrain = net.training + try: + net.eval() + yield net + finally: + if istrain: + net.train() + + +def try_to_cuda(t: Any) -> Any: + """ + Try to move the input variable `t` to a cuda device. + + Args: + t: Input. + + Returns: + t_cuda: `t` moved to a cuda device, if supported. + """ + try: + t = t.cuda() + except AttributeError: + pass + return t + + +def try_to_cpu(t: Any) -> Any: + """ + Try to move the input variable `t` to a cpu device. + + Args: + t: Input. + + Returns: + t_cpu: `t` moved to a cpu device, if supported. + """ + try: + t = t.cpu() + except AttributeError: + pass + return t + + +def dict_to_cuda(batch: Dict[Any, Any]) -> Dict[Any, Any]: + """ + Move all values in a dictionary to cuda if supported. + + Args: + batch: Input dict. + + Returns: + batch_cuda: `batch` moved to a cuda device, if supported. + """ + return {k: try_to_cuda(v) for k, v in batch.items()} + + +def dict_to_cpu(batch): + """ + Move all values in a dictionary to cpu if supported. + + Args: + batch: Input dict. + + Returns: + batch_cpu: `batch` moved to a cpu device, if supported. + """ + return {k: try_to_cpu(v) for k, v in batch.items()} + + +def dataclass_to_cuda_(obj): + """ + Move all contents of a dataclass to cuda inplace if supported. + + Args: + batch: Input dataclass. + + Returns: + batch_cuda: `batch` moved to a cuda device, if supported. + """ + for f in dataclasses.fields(obj): + setattr(obj, f.name, try_to_cuda(getattr(obj, f.name))) + return obj + + +def dataclass_to_cpu_(obj): + """ + Move all contents of a dataclass to cpu inplace if supported. + + Args: + batch: Input dataclass. + + Returns: + batch_cuda: `batch` moved to a cpu device, if supported. + """ + for f in dataclasses.fields(obj): + setattr(obj, f.name, try_to_cpu(getattr(obj, f.name))) + return obj + + +# TODO: test it +def cat_dataclass(batch, tensor_collator: Callable): + """ + Concatenate all fields of a list of dataclasses `batch` to a single + dataclass object using `tensor_collator`. + + Args: + batch: Input list of dataclasses. + + Returns: + concatenated_batch: All elements of `batch` concatenated to a single + dataclass object. + tensor_collator: The function used to concatenate tensor fields. + """ + + elem = batch[0] + collated = {} + + for f in dataclasses.fields(elem): + elem_f = getattr(elem, f.name) + if elem_f is None: + collated[f.name] = None + elif torch.is_tensor(elem_f): + collated[f.name] = tensor_collator([getattr(e, f.name) for e in batch]) + elif dataclasses.is_dataclass(elem_f): + collated[f.name] = cat_dataclass( + [getattr(e, f.name) for e in batch], tensor_collator + ) + elif isinstance(elem_f, collections.abc.Mapping): + collated[f.name] = { + k: ( + tensor_collator([getattr(e, f.name)[k] for e in batch]) + if elem_f[k] is not None + else None + ) + for k in elem_f + } + else: + raise ValueError("Unsupported field type for concatenation") + + return type(elem)(**collated) + + +def recursive_visitor(it: Iterable[Any]) -> Iterator[Any]: + for x in it: + if isinstance(x, Iterable) and not isinstance(x, (str, bytes)): + yield from recursive_visitor(x) + else: + yield x + + +def get_inlier_indicators( + tensor: torch.Tensor, dim: int, outlier_rate: float +) -> torch.Tensor: + remove_elements = int(min(outlier_rate, 1.0) * tensor.shape[dim] / 2) + hi = torch.topk(tensor, remove_elements, dim=dim).indices.tolist() + lo = torch.topk(-tensor, remove_elements, dim=dim).indices.tolist() + remove_indices = set(recursive_visitor([hi, lo])) + keep_indices = tensor.new_ones(tensor.shape[dim : dim + 1], dtype=torch.bool) + keep_indices[list(remove_indices)] = False + return keep_indices + + +class Timer: + """ + A simple class for timing execution. + + Example:: + + with Timer(): + print("This print statement is timed.") + + """ + + def __init__(self, name="timer", quiet=False): + self.name = name + self.quiet = quiet + + def __enter__(self): + self.start = time.time() + return self + + def __exit__(self, *args): + self.end = time.time() + self.interval = self.end - self.start + if not self.quiet: + print("%20s: %1.6f sec" % (self.name, self.interval)) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/video_writer.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/video_writer.py new file mode 100644 index 0000000000000000000000000000000000000000..867640712f35a3a280dd2f776b9e31ec8177796c --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/video_writer.py @@ -0,0 +1,181 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import os +import shutil +import subprocess +import tempfile +import warnings +from typing import Optional, Tuple, Union + +import matplotlib +import matplotlib.pyplot as plt +import numpy as np +from PIL import Image + +_DEFAULT_FFMPEG = os.environ.get("FFMPEG", "ffmpeg") + +matplotlib.use("Agg") + + +class VideoWriter: + """ + A class for exporting videos. + """ + + def __init__( + self, + cache_dir: Optional[str] = None, + ffmpeg_bin: str = _DEFAULT_FFMPEG, + out_path: str = "/tmp/video.mp4", + fps: int = 20, + output_format: str = "visdom", + rmdir_allowed: bool = False, + **kwargs, + ) -> None: + """ + Args: + cache_dir: A directory for storing the video frames. If `None`, + a temporary directory will be used. + ffmpeg_bin: The path to an `ffmpeg` executable. + out_path: The path to the output video. + fps: The speed of the generated video in frames-per-second. + output_format: Format of the output video. Currently only `"visdom"` + is supported. + rmdir_allowed: If `True` delete and create `cache_dir` in case + it is not empty. + """ + self.rmdir_allowed = rmdir_allowed + self.output_format = output_format + self.fps = fps + self.out_path = out_path + self.cache_dir = cache_dir + self.ffmpeg_bin = ffmpeg_bin + self.frames = [] + self.regexp = "frame_%08d.png" + self.frame_num = 0 + + if self.cache_dir is not None: + self.tmp_dir = None + if os.path.isdir(self.cache_dir): + if rmdir_allowed: + shutil.rmtree(self.cache_dir) + else: + warnings.warn( + f"Warning: cache directory not empty ({self.cache_dir})." + ) + os.makedirs(self.cache_dir, exist_ok=True) + else: + self.tmp_dir = tempfile.TemporaryDirectory() + self.cache_dir = self.tmp_dir.name + + def write_frame( + self, + frame: Union[matplotlib.figure.Figure, np.ndarray, Image.Image, str], + resize: Optional[Union[float, Tuple[int, int]]] = None, + ) -> None: + """ + Write a frame to the video. + + Args: + frame: An object containing the frame image. + resize: Either a floating defining the image rescaling factor + or a 2-tuple defining the size of the output image. + """ + + # pyre-fixme[6]: For 1st argument expected `Union[PathLike[str], str]` but + # got `Optional[str]`. + outfile = os.path.join(self.cache_dir, self.regexp % self.frame_num) + + if isinstance(frame, matplotlib.figure.Figure): + plt.savefig(outfile) + im = Image.open(outfile) + elif isinstance(frame, np.ndarray): + if frame.dtype in (np.float64, np.float32, float): + frame = (np.transpose(frame, (1, 2, 0)) * 255.0).astype(np.uint8) + im = Image.fromarray(frame) + elif isinstance(frame, Image.Image): + im = frame + elif isinstance(frame, str): + im = Image.open(frame).convert("RGB") + else: + raise ValueError("Cant convert type %s" % str(type(frame))) + + if im is not None: + if resize is not None: + if isinstance(resize, float): + resize = [int(resize * s) for s in im.size] + else: + resize = im.size + # make sure size is divisible by 2 + resize = tuple([resize[i] + resize[i] % 2 for i in (0, 1)]) + # pyre-fixme[16]: Module `Image` has no attribute `ANTIALIAS`. + im = im.resize(resize, Image.ANTIALIAS) + im.save(outfile) + + self.frames.append(outfile) + self.frame_num += 1 + + def get_video(self, quiet: bool = True) -> str: + """ + Generate the video from the written frames. + + Args: + quiet: If `True`, suppresses logging messages. + + Returns: + video_path: The path to the generated video if any frames were added. + Otherwise returns an empty string. + """ + if self.frame_num == 0: + return "" + + # pyre-fixme[6]: For 1st argument expected `Union[PathLike[str], str]` but + # got `Optional[str]`. + regexp = os.path.join(self.cache_dir, self.regexp) + + if shutil.which(self.ffmpeg_bin) is None: + raise ValueError( + f"Cannot find ffmpeg as `{self.ffmpeg_bin}`. " + + "Please set FFMPEG in the environment or ffmpeg_bin on this class." + ) + + if self.output_format == "visdom": # works for ppt too + args = [ + self.ffmpeg_bin, + "-r", + str(self.fps), + "-i", + regexp, + "-vcodec", + "h264", + "-f", + "mp4", + "-y", + "-crf", + "18", + "-b", + "2000k", + "-pix_fmt", + "yuv420p", + self.out_path, + ] + if quiet: + subprocess.check_call( + args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + else: + subprocess.check_call(args) + else: + raise ValueError("no such output type %s" % str(self.output_format)) + + return self.out_path + + def __del__(self) -> None: + if self.tmp_dir is not None: + self.tmp_dir.cleanup() diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/vis_utils.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/vis_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..662c138ccbb1900084daa3be1cede76501ee1b2e --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/implicitron/tools/vis_utils.py @@ -0,0 +1,189 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import logging +from typing import Any, Dict, Optional, Tuple, TYPE_CHECKING + +import torch + +if TYPE_CHECKING: + from visdom import Visdom + + +logger = logging.getLogger(__name__) + + +def get_visdom_env(visdom_env: str, exp_dir: str) -> str: + """ + Parse out visdom environment name from the input config. + + Args: + visdom_env: Name of the wisdom environment, could be empty string. + exp_dir: Root experiment directory. + + Returns: + visdom_env: The name of the visdom environment. If the given visdom_env is + empty, return the name of the bottom directory in exp_dir. + """ + if len(visdom_env) == 0: + visdom_env = exp_dir.split("/")[-1] + else: + visdom_env = visdom_env + return visdom_env + + +# TODO: a proper singleton +_viz_singleton = None + + +def get_visdom_connection( + server: str = "http://localhost", + port: int = 8097, +) -> Optional["Visdom"]: + """ + Obtain a connection to a visdom server if visdom is installed. + + Args: + server: Server address. + port: Server port. + + Returns: + connection: The connection object. + """ + try: + from visdom import Visdom + except ImportError: + logger.debug("Cannot load visdom") + return None + + if server == "None": + return None + + global _viz_singleton + if _viz_singleton is None: + _viz_singleton = Visdom(server=server, port=port) + return _viz_singleton + + +def visualize_basics( + viz: "Visdom", + preds: Dict[str, Any], + visdom_env_imgs: str, + title: str = "", + visualize_preds_keys: Tuple[str, ...] = ( + "image_rgb", + "images_render", + "fg_probability", + "masks_render", + "depths_render", + "depth_map", + ), + store_history: bool = False, +) -> None: + """ + Visualize basic outputs of a `GenericModel` to visdom. + + Args: + viz: The visdom object. + preds: A dictionary containing `GenericModel` outputs. + visdom_env_imgs: Target visdom environment name. + title: The title of produced visdom window. + visualize_preds_keys: The list of keys of `preds` for visualization. + store_history: Store the history buffer in visdom windows. + """ + imout = {} + for k in visualize_preds_keys: + if k not in preds or preds[k] is None: + logger.info(f"cant show {k}") + continue + v = preds[k].cpu().detach().clone() + if k.startswith("depth"): + # divide by 95th percentile + normfac = ( + v.view(v.shape[0], -1) + .topk(k=int(0.05 * (v.numel() // v.shape[0])), dim=-1) + .values[:, -1] + ) + v = v / normfac[:, None, None, None].clamp(1e-4) + if v.shape[1] == 1: + v = v.repeat(1, 3, 1, 1) + v = torch.nn.functional.interpolate( + v, + scale_factor=( + 600.0 + if ( + "_eval" in visdom_env_imgs + and k in ("images_render", "depths_render") + ) + else 200.0 + ) + / v.shape[2], + mode="bilinear", + ) + imout[k] = v + + # TODO: handle errors on the outside + try: + imout = {"all": torch.cat(list(imout.values()), dim=2)} + except RuntimeError as e: + print("cant cat!", e.args) + + for k, v in imout.items(): + viz.images( + v.clamp(0.0, 1.0), + win=k, + env=visdom_env_imgs, + opts={"title": title + "_" + k, "store_history": store_history}, + ) + + +def make_depth_image( + depths: torch.Tensor, + masks: torch.Tensor, + max_quantile: float = 0.98, + min_quantile: float = 0.02, + min_out_depth: float = 0.1, + max_out_depth: float = 0.9, +) -> torch.Tensor: + """ + Convert a batch of depth maps to a grayscale image. + + Args: + depths: A tensor of shape `(B, 1, H, W)` containing a batch of depth maps. + masks: A tensor of shape `(B, 1, H, W)` containing a batch of foreground masks. + max_quantile: The quantile of the input depth values which will + be mapped to `max_out_depth`. + min_quantile: The quantile of the input depth values which will + be mapped to `min_out_depth`. + min_out_depth: The minimal value in each depth map will be assigned this color. + max_out_depth: The maximal value in each depth map will be assigned this color. + + Returns: + depth_image: A tensor of shape `(B, 1, H, W)` a batch of grayscale + depth images. + """ + normfacs = [] + for d, m in zip(depths, masks): + ok = (d.view(-1) > 1e-6) * (m.view(-1) > 0.5) + if ok.sum() <= 1: + logger.info("empty depth!") + normfacs.append(torch.zeros(2).type_as(depths)) + continue + dok = d.view(-1)[ok].view(-1) + _maxk = max(int(round((1 - max_quantile) * (dok.numel()))), 1) + _mink = max(int(round(min_quantile * (dok.numel()))), 1) + normfac_max = dok.topk(k=_maxk, dim=-1).values[-1] + normfac_min = dok.topk(k=_mink, dim=-1, largest=False).values[-1] + normfacs.append(torch.stack([normfac_min, normfac_max])) + normfacs = torch.stack(normfacs) + _min, _max = (normfacs[:, 0].view(-1, 1, 1, 1), normfacs[:, 1].view(-1, 1, 1, 1)) + depths = (depths - _min) / (_max - _min).clamp(1e-4) + depths = ( + (depths * (max_out_depth - min_out_depth) + min_out_depth) * masks.float() + ).clamp(0.0, 1.0) + return depths diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/__init__.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8764c6011e8eba806ea7b755d0bfb81dfd816a21 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +from .obj_io import load_obj, load_objs_as_meshes, save_obj +from .pluggable import IO +from .ply_io import load_ply, save_ply + + +__all__ = [k for k in globals().keys() if not k.startswith("_")] diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/experimental_gltf_io.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/experimental_gltf_io.py new file mode 100644 index 0000000000000000000000000000000000000000..937147cebfebdd7ec6df570bfa58ecef216d451c --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/experimental_gltf_io.py @@ -0,0 +1,864 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +""" +This module implements loading meshes from glTF 2 assets stored in a +GLB container file or a glTF JSON file with embedded binary data. +It is experimental. + +The module provides a MeshFormatInterpreter called +MeshGlbFormat which must be used explicitly. +e.g. + +.. code-block:: python + + from pytorch3d.io import IO + from pytorch3d.io.experimental_gltf_io import MeshGlbFormat + + io = IO() + io.register_meshes_format(MeshGlbFormat()) + io.load_mesh(...) + +This implementation is quite restricted in what it supports. + + - It does not try to validate the input against the standard. + - It loads the default scene only. + - Only triangulated geometry is supported. + - The geometry of all meshes of the entire scene is aggregated into a single mesh. + Use `load_meshes()` instead to get un-aggregated (but transformed) ones. + - All material properties are ignored except for either vertex color, baseColorTexture + or baseColorFactor. If available, one of these (in this order) is exclusively + used which does not match the semantics of the standard. +""" + +import json +import struct +import warnings +from base64 import b64decode +from collections import defaultdict, deque +from enum import IntEnum +from io import BytesIO +from typing import Any, BinaryIO, cast, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +from iopath.common.file_io import PathManager +from PIL import Image +from pytorch3d.io.utils import _open_file, PathOrStr +from pytorch3d.renderer.mesh import TexturesBase, TexturesUV, TexturesVertex +from pytorch3d.structures import join_meshes_as_scene, Meshes +from pytorch3d.transforms import quaternion_to_matrix, Transform3d + +from .pluggable_formats import endswith, MeshFormatInterpreter + + +_GLTF_MAGIC = 0x46546C67 +_JSON_CHUNK_TYPE = 0x4E4F534A +_BINARY_CHUNK_TYPE = 0x004E4942 +_DATA_URI_PREFIX = "data:application/octet-stream;base64," + + +class _PrimitiveMode(IntEnum): + POINTS = 0 + LINES = 1 + LINE_LOOP = 2 + LINE_STRIP = 3 + TRIANGLES = 4 + TRIANGLE_STRIP = 5 + TRIANGLE_FAN = 6 + + +class _ComponentType(IntEnum): + BYTE = 5120 + UNSIGNED_BYTE = 5121 + SHORT = 5122 + UNSIGNED_SHORT = 5123 + UNSIGNED_INT = 5125 + FLOAT = 5126 + + +_ITEM_TYPES: Dict[int, Any] = { + 5120: np.int8, + 5121: np.uint8, + 5122: np.int16, + 5123: np.uint16, + 5125: np.uint32, + 5126: np.float32, +} + + +_ElementShape = Union[Tuple[int], Tuple[int, int]] +_ELEMENT_SHAPES: Dict[str, _ElementShape] = { + "SCALAR": (1,), + "VEC2": (2,), + "VEC3": (3,), + "VEC4": (4,), + "MAT2": (2, 2), + "MAT3": (3, 3), + "MAT4": (4, 4), +} + +_DTYPE_BYTES: Dict[Any, int] = { + np.int8: 1, + np.uint8: 1, + np.int16: 2, + np.uint16: 2, + np.uint32: 4, + np.float32: 4, +} + + +class _TargetType(IntEnum): + ARRAY_BUFFER = 34962 + ELEMENT_ARRAY_BUFFER = 34963 + + +class OurEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, np.int64): + return str(obj) + return super(OurEncoder, self).default(obj) + + +def _read_header(stream: BinaryIO) -> Optional[Tuple[int, int]]: + header = stream.read(12) + magic, version, length = struct.unpack(" Optional[Tuple[Dict[str, Any], np.ndarray]]: + """ + Get the json header and the binary data from a + GLB file. + """ + json_data = None + binary_data = None + + while stream.tell() < length: + chunk_header = stream.read(8) + chunk_length, chunk_type = struct.unpack(" Transform3d: + """ + Convert a transform from the json data in to a PyTorch3D + Transform3d format. + """ + array = node.get("matrix") + if array is not None: # Stored in column-major order + M = np.array(array, dtype=np.float32).reshape(4, 4, order="F") + return Transform3d(matrix=torch.from_numpy(M)) + + out = Transform3d() + + # Given some of (scale/rotation/translation), we do them in that order to + # get points in to the world space. + # See https://github.com/KhronosGroup/glTF/issues/743 . + + array = node.get("scale", None) + if array is not None: + scale_vector = torch.FloatTensor(array) + out = out.scale(scale_vector[None]) + + # Rotation quaternion (x, y, z, w) where w is the scalar + array = node.get("rotation", None) + if array is not None: + x, y, z, w = array + # We negate w. This is equivalent to inverting the rotation. + # This is needed as quaternion_to_matrix makes a matrix which + # operates on column vectors, whereas Transform3d wants a + # matrix which operates on row vectors. + rotation_quaternion = torch.FloatTensor([-w, x, y, z]) + rotation_matrix = quaternion_to_matrix(rotation_quaternion) + out = out.rotate(R=rotation_matrix) + + array = node.get("translation", None) + if array is not None: + translation_vector = torch.FloatTensor(array) + out = out.translate(x=translation_vector[None]) + + return out + + +class _GLTFLoader: + def __init__(self, stream: BinaryIO) -> None: + self._json_data = None + # Map from buffer index to (decoded) binary data + self._binary_data = {} + + version_and_length = _read_header(stream) + if version_and_length is None: # GLTF + stream.seek(0) + json_data = json.load(stream) + else: # GLB + version, length = version_and_length + if version != 2: + warnings.warn("Unsupported version") + return + json_and_binary_data = _read_chunks(stream, length) + if json_and_binary_data is None: + raise ValueError("Data not found") + json_data, binary_data = json_and_binary_data + self._binary_data[0] = binary_data + + self._json_data = json_data + self._accessors = json_data.get("accessors", []) + self._buffer_views = json_data.get("bufferViews", []) + self._buffers = json_data.get("buffers", []) + self._texture_map_images = {} + + def _access_image(self, image_index: int) -> np.ndarray: + """ + Get the data for an image from the file. This is only called + by _get_texture_map_image which caches it. + """ + + image_json = self._json_data["images"][image_index] + buffer_view = self._buffer_views[image_json["bufferView"]] + if "byteStride" in buffer_view: + raise NotImplementedError("strided buffer views") + + length = buffer_view["byteLength"] + offset = buffer_view.get("byteOffset", 0) + + binary_data = self.get_binary_data(buffer_view["buffer"]) + bytesio = BytesIO(binary_data[offset : offset + length].tobytes()) + with Image.open(bytesio) as f: + array = np.array(f) + if array.dtype == np.uint8: + return array.astype(np.float32) / 255.0 + else: + return array + + def _get_texture_map_image(self, image_index: int) -> torch.Tensor: + """ + Return a texture map image as a torch tensor. + Calling this function repeatedly with the same arguments returns + the very same tensor, this allows a memory optimization to happen + later in TexturesUV.join_scene. + Any alpha channel is ignored. + """ + im = self._texture_map_images.get(image_index) + if im is not None: + return im + + im = torch.from_numpy(self._access_image(image_index))[:, :, :3] + self._texture_map_images[image_index] = im + return im + + def _access_data(self, accessor_index: int) -> np.ndarray: + """ + Get the raw data from an accessor as a numpy array. + """ + accessor = self._accessors[accessor_index] + + buffer_view_index = accessor.get("bufferView") + # Undefined buffer view (all zeros) are not (yet) supported + if buffer_view_index is None: + raise NotImplementedError("Undefined buffer view") + + accessor_byte_offset = accessor.get("byteOffset", 0) + component_type = accessor["componentType"] + element_count = accessor["count"] + element_type = accessor["type"] + + # Sparse accessors are not (yet) supported + if accessor.get("sparse") is not None: + raise NotImplementedError("Sparse Accessors") + + buffer_view = self._buffer_views[buffer_view_index] + buffer_index = buffer_view["buffer"] + buffer_byte_length = buffer_view["byteLength"] + element_byte_offset = buffer_view.get("byteOffset", 0) + element_byte_stride = buffer_view.get("byteStride", 0) + if element_byte_stride != 0 and element_byte_stride < 4: + raise ValueError("Stride is too small.") + if element_byte_stride > 252: + raise ValueError("Stride is too big.") + + element_shape = _ELEMENT_SHAPES[element_type] + item_type = _ITEM_TYPES[component_type] + item_dtype = np.dtype(item_type) + item_count = np.prod(element_shape) + item_size = item_dtype.itemsize + size = element_count * item_count * item_size + if size > buffer_byte_length: + raise ValueError("Buffer did not have enough data for the accessor") + + buffer_ = self._buffers[buffer_index] + binary_data = self.get_binary_data(buffer_index) + if len(binary_data) < buffer_["byteLength"]: + raise ValueError("Not enough binary data for the buffer") + + if element_byte_stride == 0: + element_byte_stride = item_size * item_count + # The same buffer can store interleaved elements + if element_byte_stride < item_size * item_count: + raise ValueError("Items should not overlap") + + dtype = np.dtype( + { + "names": ["element"], + "formats": [str(element_shape) + item_dtype.str], + "offsets": [0], + "itemsize": element_byte_stride, + } + ) + + byte_offset = accessor_byte_offset + element_byte_offset + if byte_offset % item_size != 0: + raise ValueError("Misaligned data") + byte_length = element_count * element_byte_stride + buffer_view = binary_data[byte_offset : byte_offset + byte_length].view(dtype)[ + "element" + ] + + # Convert matrix data from column-major (OpenGL) to row-major order + if element_type in ("MAT2", "MAT3", "MAT4"): + buffer_view = np.transpose(buffer_view, (0, 2, 1)) + + return buffer_view + + def _get_primitive_attribute( + self, primitive_attributes: Dict[str, Any], key: str, dtype + ) -> Optional[np.ndarray]: + accessor_index = primitive_attributes.get(key) + if accessor_index is None: + return None + primitive_attribute = self._access_data(accessor_index) + if key == "JOINTS_0": + pass + elif dtype == np.uint8: + primitive_attribute /= 255.0 + elif dtype == np.uint16: + primitive_attribute /= 65535.0 + else: + if dtype != np.float32: + raise ValueError("Unexpected data type") + primitive_attribute = primitive_attribute.astype(dtype) + return primitive_attribute + + def get_binary_data(self, buffer_index: int): + """ + Get the binary data from a buffer as a 1D numpy array of bytes. + This is implemented for explicit uri data buffers or the main GLB data + segment. + """ + buffer_ = self._buffers[buffer_index] + binary_data = self._binary_data.get(buffer_index) + if binary_data is None: # Lazily decode binary data + uri = buffer_.get("uri") + if not uri.startswith(_DATA_URI_PREFIX): + raise NotImplementedError("Unexpected URI type") + binary_data = b64decode(uri[len(_DATA_URI_PREFIX) :]) + binary_data = np.frombuffer(binary_data, dtype=np.uint8) + self._binary_data[buffer_index] = binary_data + return binary_data + + def get_texture_for_mesh( + self, primitive: Dict[str, Any], indices: torch.Tensor + ) -> Optional[TexturesBase]: + """ + Get the texture object representing the given mesh primitive. + + Args: + primitive: the mesh primitive being loaded. + indices: the face indices of the mesh + """ + attributes = primitive["attributes"] + vertex_colors = self._get_primitive_attribute(attributes, "COLOR_0", np.float32) + if vertex_colors is not None: + return TexturesVertex([torch.from_numpy(vertex_colors)]) + + vertex_texcoords_0 = self._get_primitive_attribute( + attributes, "TEXCOORD_0", np.float32 + ) + if vertex_texcoords_0 is not None: + verts_uvs = torch.from_numpy(vertex_texcoords_0) + verts_uvs[:, 1] = 1 - verts_uvs[:, -1] + faces_uvs = indices + material_index = primitive.get("material", 0) + material = self._json_data["materials"][material_index] + material_roughness = material["pbrMetallicRoughness"] + if "baseColorTexture" in material_roughness: + texture_index = material_roughness["baseColorTexture"]["index"] + texture_json = self._json_data["textures"][texture_index] + # Todo - include baseColorFactor when also given + # Todo - look at the sampler + image_index = texture_json["source"] + map = self._get_texture_map_image(image_index) + elif "baseColorFactor" in material_roughness: + # Constant color? + map = torch.FloatTensor(material_roughness["baseColorFactor"])[ + None, None, :3 + ] + texture = TexturesUV( + # pyre-fixme[61]: `map` may not be initialized here. + maps=[map], # alpha channel ignored + faces_uvs=[faces_uvs], + verts_uvs=[verts_uvs], + ) + return texture + + return None + + def load(self, include_textures: bool) -> List[Tuple[Optional[str], Meshes]]: + """ + Attempt to load all the meshes making up the default scene from + the file as a list of possibly-named Meshes objects. + + Args: + include_textures: Whether to try loading textures. + + Returns: + Meshes object containing one mesh. + """ + if self._json_data is None: + raise ValueError("Initialization problem") + + # This loads the default scene from the file. + # This is usually the only one. + # It is possible to have multiple scenes, in which case + # you could choose another here instead of taking the default. + scene_index = self._json_data.get("scene") + + if scene_index is None: + raise ValueError("Default scene is not specified.") + + scene = self._json_data["scenes"][scene_index] + nodes = self._json_data.get("nodes", []) + meshes = self._json_data.get("meshes", []) + root_node_indices = scene["nodes"] + + mesh_transform = Transform3d() + names_meshes_list: List[Tuple[Optional[str], Meshes]] = [] + + # Keep track and apply the transform of the scene node to mesh vertices + Q = deque([(Transform3d(), node_index) for node_index in root_node_indices]) + + while Q: + parent_transform, current_node_index = Q.popleft() + + current_node = nodes[current_node_index] + + transform = _make_node_transform(current_node) + current_transform = transform.compose(parent_transform) + + if "mesh" in current_node: + mesh_index = current_node["mesh"] + mesh = meshes[mesh_index] + mesh_name = mesh.get("name", None) + mesh_transform = current_transform + + for primitive in mesh["primitives"]: + attributes = primitive["attributes"] + accessor_index = attributes["POSITION"] + positions = torch.from_numpy( + self._access_data(accessor_index).copy() + ) + positions = mesh_transform.transform_points(positions) + + mode = primitive.get("mode", _PrimitiveMode.TRIANGLES) + if mode != _PrimitiveMode.TRIANGLES: + raise NotImplementedError("Non triangular meshes") + + if "indices" in primitive: + accessor_index = primitive["indices"] + indices = self._access_data(accessor_index).astype(np.int64) + else: + indices = np.arange(0, len(positions), dtype=np.int64) + indices = torch.from_numpy(indices.reshape(-1, 3)) + + texture = None + if include_textures: + texture = self.get_texture_for_mesh(primitive, indices) + + mesh_obj = Meshes( + verts=[positions], faces=[indices], textures=texture + ) + names_meshes_list.append((mesh_name, mesh_obj)) + + if "children" in current_node: + children_node_indices = current_node["children"] + Q.extend( + [ + (current_transform, node_index) + for node_index in children_node_indices + ] + ) + + return names_meshes_list + + +def load_meshes( + path: PathOrStr, + path_manager: PathManager, + include_textures: bool = True, +) -> List[Tuple[Optional[str], Meshes]]: + """ + Loads all the meshes from the default scene in the given GLB file. + and returns them separately. + + Args: + path: path to read from + path_manager: PathManager object for interpreting the path + include_textures: whether to load textures + + Returns: + List of (name, mesh) pairs, where the name is the optional name property + from the GLB file, or None if it is absent, and the mesh is a Meshes + object containing one mesh. + """ + with _open_file(path, path_manager, "rb") as f: + loader = _GLTFLoader(cast(BinaryIO, f)) + names_meshes_list = loader.load(include_textures=include_textures) + return names_meshes_list + + +class _GLTFWriter: + def __init__(self, data: Meshes, buffer_stream: BinaryIO) -> None: + self._json_data = defaultdict(list) + self.mesh = data + self.buffer_stream = buffer_stream + + # initialize json with one scene and one node + scene_index = 0 + # pyre-fixme[6]: Incompatible parameter type + self._json_data["scene"] = scene_index + self._json_data["scenes"].append({"nodes": [scene_index]}) + self._json_data["asset"] = {"version": "2.0"} + node = {"name": "Node", "mesh": 0} + self._json_data["nodes"].append(node) + + # mesh primitives + meshes = defaultdict(list) + # pyre-fixme[6]: Incompatible parameter type + meshes["name"] = "Node-Mesh" + if isinstance(self.mesh.textures, TexturesVertex): + primitives = { + "attributes": {"POSITION": 0, "COLOR_0": 2}, + "indices": 1, + "mode": _PrimitiveMode.TRIANGLES, + } + elif isinstance(self.mesh.textures, TexturesUV): + primitives = { + "attributes": {"POSITION": 0, "TEXCOORD_0": 2}, + "indices": 1, + "mode": _PrimitiveMode.TRIANGLES, + "material": 0, + } + else: + primitives = { + "attributes": {"POSITION": 0}, + "indices": 1, + "mode": _PrimitiveMode.TRIANGLES, + } + + meshes["primitives"].append(primitives) + self._json_data["meshes"].append(meshes) + + # default material + material = { + "name": "material_1", + "pbrMetallicRoughness": { + "baseColorTexture": {"index": 0}, + "baseColorFactor": [1, 1, 1, 1], + "metallicFactor": 0, + "roughnessFactor": 0.99, + }, + "emissiveFactor": [0, 0, 0], + "alphaMode": "OPAQUE", + } + self._json_data["materials"].append(material) + + # default sampler + sampler = {"magFilter": 9729, "minFilter": 9986, "wrapS": 10497, "wrapT": 10497} + self._json_data["samplers"].append(sampler) + + # default textures + texture = {"sampler": 0, "source": 0} + self._json_data["textures"].append(texture) + + def _write_accessor_json(self, key: str) -> Tuple[int, np.ndarray]: + name = "Node-Mesh_%s" % key + byte_offset = 0 + if key == "positions": + data = self.mesh.verts_packed().cpu().numpy() + component_type = _ComponentType.FLOAT + element_type = "VEC3" + buffer_view = 0 + element_min = list(map(float, np.min(data, axis=0))) + element_max = list(map(float, np.max(data, axis=0))) + byte_per_element = 3 * _DTYPE_BYTES[_ITEM_TYPES[_ComponentType.FLOAT]] + elif key == "texcoords": + component_type = _ComponentType.FLOAT + data = self.mesh.textures.verts_uvs_list()[0].cpu().numpy() + data[:, 1] = 1 - data[:, -1] # flip y tex-coordinate + element_type = "VEC2" + buffer_view = 2 + element_min = list(map(float, np.min(data, axis=0))) + element_max = list(map(float, np.max(data, axis=0))) + byte_per_element = 2 * _DTYPE_BYTES[_ITEM_TYPES[_ComponentType.FLOAT]] + elif key == "texvertices": + component_type = _ComponentType.FLOAT + data = self.mesh.textures.verts_features_list()[0].cpu().numpy() + element_type = "VEC3" + buffer_view = 2 + element_min = list(map(float, np.min(data, axis=0))) + element_max = list(map(float, np.max(data, axis=0))) + byte_per_element = 3 * _DTYPE_BYTES[_ITEM_TYPES[_ComponentType.FLOAT]] + elif key == "indices": + component_type = _ComponentType.UNSIGNED_SHORT + data = ( + self.mesh.faces_packed() + .cpu() + .numpy() + .astype(_ITEM_TYPES[component_type]) + ) + element_type = "SCALAR" + buffer_view = 1 + element_min = list(map(int, np.min(data, keepdims=True))) + element_max = list(map(int, np.max(data, keepdims=True))) + byte_per_element = ( + 3 * _DTYPE_BYTES[_ITEM_TYPES[_ComponentType.UNSIGNED_SHORT]] + ) + else: + raise NotImplementedError( + "invalid key accessor, should be one of positions, indices or texcoords" + ) + + count = int(data.shape[0]) + byte_length = count * byte_per_element + accessor_json = { + "name": name, + "componentType": component_type, + "type": element_type, + "bufferView": buffer_view, + "byteOffset": byte_offset, + "min": element_min, + "max": element_max, + "count": count * 3 if key == "indices" else count, + } + self._json_data["accessors"].append(accessor_json) + return (byte_length, data) + + def _write_bufferview(self, key: str, **kwargs): + if key not in ["positions", "texcoords", "texvertices", "indices"]: + raise ValueError( + "key must be one of positions, texcoords, texvertices or indices" + ) + + bufferview = { + "name": "bufferView_%s" % key, + "buffer": 0, + } + target = _TargetType.ARRAY_BUFFER + if key == "positions": + byte_per_element = 3 * _DTYPE_BYTES[_ITEM_TYPES[_ComponentType.FLOAT]] + bufferview["byteStride"] = int(byte_per_element) + elif key == "texcoords": + byte_per_element = 2 * _DTYPE_BYTES[_ITEM_TYPES[_ComponentType.FLOAT]] + target = _TargetType.ARRAY_BUFFER + bufferview["byteStride"] = int(byte_per_element) + elif key == "texvertices": + byte_per_element = 3 * _DTYPE_BYTES[_ITEM_TYPES[_ComponentType.FLOAT]] + target = _TargetType.ELEMENT_ARRAY_BUFFER + bufferview["byteStride"] = int(byte_per_element) + elif key == "indices": + byte_per_element = ( + 3 * _DTYPE_BYTES[_ITEM_TYPES[_ComponentType.UNSIGNED_SHORT]] + ) + target = _TargetType.ELEMENT_ARRAY_BUFFER + + bufferview["target"] = target + bufferview["byteOffset"] = kwargs.get("offset") + bufferview["byteLength"] = kwargs.get("byte_length") + self._json_data["bufferViews"].append(bufferview) + + def _write_image_buffer(self, **kwargs) -> Tuple[int, bytes]: + image_np = self.mesh.textures.maps_list()[0].cpu().numpy() + image_array = (image_np * 255.0).astype(np.uint8) + im = Image.fromarray(image_array) + with BytesIO() as f: + im.save(f, format="PNG") + image_data = f.getvalue() + + image_data_byte_length = len(image_data) + bufferview_image = { + "buffer": 0, + } + bufferview_image["byteOffset"] = kwargs.get("offset") + bufferview_image["byteLength"] = image_data_byte_length + self._json_data["bufferViews"].append(bufferview_image) + + image = {"name": "texture", "mimeType": "image/png", "bufferView": 3} + self._json_data["images"].append(image) + return (image_data_byte_length, image_data) + + def save(self): + # check validity of mesh + if self.mesh.verts_packed() is None or self.mesh.faces_packed() is None: + raise ValueError("invalid mesh to save, verts or face indices are empty") + + # accessors for positions, texture uvs and face indices + pos_byte, pos_data = self._write_accessor_json("positions") + idx_byte, idx_data = self._write_accessor_json("indices") + include_textures = False + if self.mesh.textures is not None: + if hasattr(self.mesh.textures, "verts_features_list"): + tex_byte, tex_data = self._write_accessor_json("texvertices") + include_textures = True + texcoords = False + elif self.mesh.textures.verts_uvs_list()[0] is not None: + tex_byte, tex_data = self._write_accessor_json("texcoords") + include_textures = True + texcoords = True + + # bufferViews for positions, texture coords and indices + byte_offset = 0 + self._write_bufferview("positions", byte_length=pos_byte, offset=byte_offset) + byte_offset += pos_byte + + self._write_bufferview("indices", byte_length=idx_byte, offset=byte_offset) + byte_offset += idx_byte + + if include_textures: + if texcoords: + self._write_bufferview( + "texcoords", byte_length=tex_byte, offset=byte_offset + ) + else: + self._write_bufferview( + "texvertices", byte_length=tex_byte, offset=byte_offset + ) + byte_offset += tex_byte + + # image bufferView + include_image = False + if self.mesh.textures is not None and hasattr(self.mesh.textures, "maps_list"): + include_image = True + image_byte, image_data = self._write_image_buffer(offset=byte_offset) + byte_offset += image_byte + + # buffers + self._json_data["buffers"].append({"byteLength": int(byte_offset)}) + + # organize into a glb + json_bytes = bytes(json.dumps(self._json_data, cls=OurEncoder), "utf-8") + json_length = len(json_bytes) + + # write header + version = 2 + total_header_length = 28 # (file header = 12) + 2 * (chunk header = 8) + file_length = json_length + byte_offset + total_header_length + header = struct.pack(" None: + self.known_suffixes = (".glb",) + + def read( + self, + path: PathOrStr, + include_textures: bool, + device, + path_manager: PathManager, + **kwargs, + ) -> Optional[Meshes]: + if not endswith(path, self.known_suffixes): + return None + + names_meshes_list = load_meshes( + path=path, + path_manager=path_manager, + include_textures=include_textures, + ) + + meshes_list = [mesh for name, mesh in names_meshes_list] + mesh = join_meshes_as_scene(meshes_list) + return mesh.to(device) + + def save( + self, + data: Meshes, + path: PathOrStr, + path_manager: PathManager, + binary: Optional[bool], + **kwargs, + ) -> bool: + """ + Writes all the meshes from the default scene to GLB file. + + Args: + data: meshes to save + path: path of the GLB file to write into + path_manager: PathManager object for interpreting the path + + Return True if saving succeeds and False otherwise + """ + + if not endswith(path, self.known_suffixes): + return False + + with _open_file(path, path_manager, "wb") as f: + writer = _GLTFWriter(data, cast(BinaryIO, f)) + writer.save() + return True diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/mtl_io.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/mtl_io.py new file mode 100644 index 0000000000000000000000000000000000000000..bbe25c0f38f262be143c8ef8748d8a1559c265d9 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/mtl_io.py @@ -0,0 +1,537 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +"""This module implements utility functions for loading .mtl files and textures.""" +import os +import warnings +from typing import Dict, List, Optional, Tuple + +import numpy as np +import torch +import torch.nn.functional as F +from iopath.common.file_io import PathManager +from pytorch3d.common.compat import meshgrid_ij +from pytorch3d.common.datatypes import Device +from pytorch3d.io.utils import _open_file, _read_image + + +def make_mesh_texture_atlas( + material_properties: Dict, + texture_images: Dict, + face_material_names, + faces_uvs: torch.Tensor, + verts_uvs: torch.Tensor, + texture_size: int, + texture_wrap: Optional[str], +) -> torch.Tensor: + """ + Given properties for materials defined in the .mtl file, and the face texture uv + coordinates, construct an (F, R, R, 3) texture atlas where R is the texture_size + and F is the number of faces in the mesh. + + Args: + material_properties: dict of properties for each material. If a material + does not have any properties it will have an empty dict. + texture_images: dict of material names and texture images + face_material_names: numpy array of the material name corresponding to each + face. Faces which don't have an associated material will be an empty string. + For these faces, a uniform white texture is assigned. + faces_uvs: LongTensor of shape (F, 3,) giving the index into the verts_uvs for + each face in the mesh. + verts_uvs: FloatTensor of shape (V, 2) giving the uv coordinates for each vertex. + texture_size: the resolution of the per face texture map returned by this function. + Each face will have a texture map of shape (texture_size, texture_size, 3). + texture_wrap: string, one of ["repeat", "clamp", None] + If `texture_wrap="repeat"` for uv values outside the range [0, 1] the integer part + is ignored and a repeating pattern is formed. + If `texture_wrap="clamp"` the values are clamped to the range [0, 1]. + If None, do nothing. + + Returns: + atlas: FloatTensor of shape (F, texture_size, texture_size, 3) giving the per + face texture map. + """ + # Create an R x R texture map per face in the mesh + R = texture_size + F = faces_uvs.shape[0] + + # Initialize the per face texture map to a white color. + # TODO: allow customization of this base color? + atlas = torch.ones(size=(F, R, R, 3), dtype=torch.float32, device=faces_uvs.device) + + # Check for empty materials. + if not material_properties and not texture_images: + return atlas + + # Iterate through the material properties - not + # all materials have texture images so this is + # done first separately to the texture interpolation. + for material_name, props in material_properties.items(): + # Bool to indicate which faces use this texture map. + faces_material_ind = torch.from_numpy(face_material_names == material_name).to( + faces_uvs.device + ) + if faces_material_ind.sum() > 0: + # For these faces, update the base color to the + # diffuse material color. + if "diffuse_color" not in props: + continue + atlas[faces_material_ind, ...] = props["diffuse_color"][None, :] + + # If there are vertex texture coordinates, create an (F, 3, 2) + # tensor of the vertex textures per face. + faces_verts_uvs = verts_uvs[faces_uvs] if len(verts_uvs) > 0 else None + + # Some meshes only have material properties and no texture image. + # In this case, return the atlas here. + if faces_verts_uvs is None: + return atlas + + if texture_wrap == "repeat": + # If texture uv coordinates are outside the range [0, 1] follow + # the convention GL_REPEAT in OpenGL i.e the integer part of the coordinate + # will be ignored and a repeating pattern is formed. + # Shapenet data uses this format see: + # https://shapenet.org/qaforum/index.php?qa=15&qa_1=why-is-the-texture-coordinate-in-the-obj-file-not-in-the-range # noqa: B950 + if (faces_verts_uvs > 1).any() or (faces_verts_uvs < 0).any(): + msg = "Texture UV coordinates outside the range [0, 1]. \ + The integer part will be ignored to form a repeating pattern." + warnings.warn(msg) + faces_verts_uvs = faces_verts_uvs % 1 + elif texture_wrap == "clamp": + # Clamp uv coordinates to the [0, 1] range. + faces_verts_uvs = faces_verts_uvs.clamp(0.0, 1.0) + + # Iterate through the materials used in this mesh. Update the + # texture atlas for the faces which use this material. + # Faces without texture are white. + for material_name, image in list(texture_images.items()): + # Only use the RGB colors + if image.shape[2] == 4: + image = image[:, :, :3] + + # Reverse the image y direction + image = torch.flip(image, [0]).type_as(faces_verts_uvs) + + # Bool to indicate which faces use this texture map. + faces_material_ind = torch.from_numpy(face_material_names == material_name).to( + faces_verts_uvs.device + ) + + # Find the subset of faces which use this texture with this texture image + uvs_subset = faces_verts_uvs[faces_material_ind, :, :] + + # Update the texture atlas for the faces which use this texture. + # TODO: should the texture map values be multiplied + # by the diffuse material color (i.e. use *= as the atlas has + # been initialized to the diffuse color)?. This is + # not being done in SoftRas. + atlas[faces_material_ind, :, :] = make_material_atlas(image, uvs_subset, R) + + return atlas + + +def make_material_atlas( + image: torch.Tensor, faces_verts_uvs: torch.Tensor, texture_size: int +) -> torch.Tensor: + r""" + Given a single texture image and the uv coordinates for all the + face vertices, create a square texture map per face using + the formulation from [1]. + + For a triangle with vertices (v0, v1, v2) we can create a barycentric coordinate system + with the x axis being the vector (v0 - v2) and the y axis being the vector (v1 - v2). + The barycentric coordinates range from [0, 1] in the +x and +y direction so this creates + a triangular texture space with vertices at (0, 1), (0, 0) and (1, 0). + + The per face texture map is of shape (texture_size, texture_size, 3) + which is a square. To map a triangular texture to a square grid, each + triangle is parametrized as follows (e.g. R = texture_size = 3): + + The triangle texture is first divided into RxR = 9 subtriangles which each + map to one grid cell. The numbers in the grid cells and triangles show the mapping. + + ..code-block::python + + Triangular Texture Space: + + 1 + |\ + |6 \ + |____\ + |\ 7 |\ + |3 \ |4 \ + |____\|____\ + |\ 8 |\ 5 |\ + |0 \ |1 \ |2 \ + |____\|____\|____\ + 0 1 + + Square per face texture map: + + R ____________________ + | | | | + | 6 | 7 | 8 | + |______|______|______| + | | | | + | 3 | 4 | 5 | + |______|______|______| + | | | | + | 0 | 1 | 2 | + |______|______|______| + 0 R + + + The barycentric coordinates of each grid cell are calculated using the + xy coordinates: + + ..code-block::python + + The cartesian coordinates are: + + Grid 1: + + R ____________________ + | | | | + | 20 | 21 | 22 | + |______|______|______| + | | | | + | 10 | 11 | 12 | + |______|______|______| + | | | | + | 00 | 01 | 02 | + |______|______|______| + 0 R + + where 02 means y = 0, x = 2 + + Now consider this subset of the triangle which corresponds to + grid cells 0 and 8: + + ..code-block::python + + 1/R ________ + |\ 8 | + | \ | + | 0 \ | + |_______\| + 0 1/R + + The centroids of the triangles are: + 0: (1/3, 1/3) * 1/R + 8: (2/3, 2/3) * 1/R + + For each grid cell we can now calculate the centroid `(c_y, c_x)` + of the corresponding texture triangle: + - if `(x + y) < R`, then offset the centroid of + triangle 0 by `(y, x) * (1/R)` + - if `(x + y) > R`, then offset the centroid of + triangle 8 by `((R-1-y), (R-1-x)) * (1/R)`. + + This is equivalent to updating the portion of Grid 1 + above the diagonal, replacing `(y, x)` with `((R-1-y), (R-1-x))`: + + ..code-block::python + + R _____________________ + | | | | + | 20 | 01 | 00 | + |______|______|______| + | | | | + | 10 | 11 | 10 | + |______|______|______| + | | | | + | 00 | 01 | 02 | + |______|______|______| + 0 R + + The barycentric coordinates (w0, w1, w2) are then given by: + + ..code-block::python + + w0 = c_x + w1 = c_y + w2 = 1- w0 - w1 + + Args: + image: FloatTensor of shape (H, W, 3) + faces_verts_uvs: uv coordinates for each vertex in each face (F, 3, 2) + texture_size: int + + Returns: + atlas: a FloatTensor of shape (F, texture_size, texture_size, 3) giving a + per face texture map. + + [1] Liu et al, 'Soft Rasterizer: A Differentiable Renderer for Image-based + 3D Reasoning', ICCV 2019 + """ + R = texture_size + device = faces_verts_uvs.device + rng = torch.arange(R, device=device) + + # Meshgrid returns (row, column) i.e (Y, X) + # Change order to (X, Y) to make the grid. + Y, X = meshgrid_ij(rng, rng) + # pyre-fixme[28]: Unexpected keyword argument `axis`. + grid = torch.stack([X, Y], axis=-1) # (R, R, 2) + + # Grid cells below the diagonal: x + y < R. + below_diag = grid.sum(-1) < R + + # map a [0, R] grid -> to a [0, 1] barycentric coordinates of + # the texture triangle centroids. + bary = torch.zeros((R, R, 3), device=device) # (R, R, 3) + slc = torch.arange(2, device=device)[:, None] + # w0, w1 + bary[below_diag, slc] = ((grid[below_diag] + 1.0 / 3.0) / R).T + # w0, w1 for above diagonal grid cells. + bary[~below_diag, slc] = (((R - 1.0 - grid[~below_diag]) + 2.0 / 3.0) / R).T + # w2 = 1. - w0 - w1 + bary[..., -1] = 1 - bary[..., :2].sum(dim=-1) + + # Calculate the uv position in the image for each pixel + # in the per face texture map + # (F, 1, 1, 3, 2) * (R, R, 3, 1) -> (F, R, R, 3, 2) -> (F, R, R, 2) + uv_pos = (faces_verts_uvs[:, None, None] * bary[..., None]).sum(-2) + + # bi-linearly interpolate the textures from the images + # using the uv coordinates given by uv_pos. + textures = _bilinear_interpolation_grid_sample(image, uv_pos) + + return textures + + +def _bilinear_interpolation_vectorized( + image: torch.Tensor, grid: torch.Tensor +) -> torch.Tensor: + """ + Bi linearly interpolate the image using the uv positions in the flow-field + grid (following the naming conventions for torch.nn.functional.grid_sample). + + This implementation uses the same steps as in the SoftRasterizer CUDA kernel + for loading textures. We are keeping it for reference to make it easy to + compare if required. + + However it doesn't properly handle the out of bound values in the same way as + the grid_sample function does with the padding_mode argument. + This vectorized version requires less memory than + _bilinear_interpolation_grid_sample but is slightly slower. + + Args: + image: FloatTensor of shape (H, W, D) a single image/input tensor with D + channels. + grid: FloatTensor of shape (N, R, R, 2) giving the pixel locations of the + points at which to sample a value in the image. The grid values must + be in the range [0, 1]. u is the x direction and v is the y direction. + + Returns: + out: FloatTensor of shape (N, H, W, D) giving the interpolated + D dimensional value from image at each of the pixel locations in grid. + + """ + H, W, _ = image.shape + # Convert [0, 1] to the range [0, W-1] and [0, H-1] + grid = grid * torch.tensor([W - 1, H - 1]).type_as(grid) + weight_1 = grid - grid.int() + weight_0 = 1.0 - weight_1 + + grid_x, grid_y = grid.unbind(-1) + y0 = grid_y.to(torch.int64) + y1 = (grid_y + 1).to(torch.int64) + x0 = grid_x.to(torch.int64) + x1 = x0 + 1 + + weight_x0, weight_y0 = weight_0.unbind(-1) + weight_x1, weight_y1 = weight_1.unbind(-1) + + # Bi-linear interpolation + # griditions = [[y, x], [(y+1), x] + # [y, (x+1)], [(y+1), (x+1)]] + # weights = [[wx0*wy0, wx0*wy1], + # [wx1*wy0, wx1*wy1]] + out = ( + image[y0, x0] * (weight_x0 * weight_y0)[..., None] + + image[y1, x0] * (weight_x0 * weight_y1)[..., None] + + image[y0, x1] * (weight_x1 * weight_y0)[..., None] + + image[y1, x1] * (weight_x1 * weight_y1)[..., None] + ) + + return out + + +def _bilinear_interpolation_grid_sample( + image: torch.Tensor, grid: torch.Tensor +) -> torch.Tensor: + """ + Bi linearly interpolate the image using the uv positions in the flow-field + grid (following the conventions for torch.nn.functional.grid_sample). + + This implementation is faster than _bilinear_interpolation_vectorized but + requires more memory so can cause OOMs. If speed is an issue try this function + instead. + + Args: + image: FloatTensor of shape (H, W, D) a single image/input tensor with D + channels. + grid: FloatTensor of shape (N, R, R, 2) giving the pixel locations of the + points at which to sample a value in the image. The grid values must + be in the range [0, 1]. u is the x direction and v is the y direction. + + Returns: + out: FloatTensor of shape (N, H, W, D) giving the interpolated + D dimensional value from image at each of the pixel locations in grid. + """ + + N = grid.shape[0] + # convert [0, 1] to the range [-1, 1] expected by grid_sample. + grid = grid * 2.0 - 1.0 + image = image.permute(2, 0, 1)[None, ...].expand(N, -1, -1, -1) # (N, 3, H, W) + # Align_corners has to be set to True to match the output of the SoftRas + # cuda kernel for bilinear sampling. + out = F.grid_sample(image, grid, mode="bilinear", align_corners=True) + return out.permute(0, 2, 3, 1) + + +MaterialProperties = Dict[str, Dict[str, torch.Tensor]] +TextureFiles = Dict[str, str] +TextureImages = Dict[str, torch.Tensor] + + +def _parse_mtl( + f: str, path_manager: PathManager, device: Device = "cpu" +) -> Tuple[MaterialProperties, TextureFiles]: + material_properties = {} + texture_files = {} + material_name = "" + + with _open_file(f, path_manager, "r") as f: + for line in f: + tokens = line.strip().split() + if not tokens: + continue + if tokens[0] == "newmtl": + material_name = tokens[1] + material_properties[material_name] = {} + elif tokens[0] == "map_Kd": + # Diffuse texture map + # Account for the case where filenames might have spaces + filename = line.strip()[7:] + texture_files[material_name] = filename + elif tokens[0] == "Kd": + # RGB diffuse reflectivity + kd = np.array(tokens[1:4]).astype(np.float32) + kd = torch.from_numpy(kd).to(device) + material_properties[material_name]["diffuse_color"] = kd + elif tokens[0] == "Ka": + # RGB ambient reflectivity + ka = np.array(tokens[1:4]).astype(np.float32) + ka = torch.from_numpy(ka).to(device) + material_properties[material_name]["ambient_color"] = ka + elif tokens[0] == "Ks": + # RGB specular reflectivity + ks = np.array(tokens[1:4]).astype(np.float32) + ks = torch.from_numpy(ks).to(device) + material_properties[material_name]["specular_color"] = ks + elif tokens[0] == "Ns": + # Specular exponent + ns = np.array(tokens[1:4]).astype(np.float32) + ns = torch.from_numpy(ns).to(device) + material_properties[material_name]["shininess"] = ns + + return material_properties, texture_files + + +def _load_texture_images( + material_names: List[str], + data_dir: str, + material_properties: MaterialProperties, + texture_files: TextureFiles, + path_manager: PathManager, +) -> Tuple[MaterialProperties, TextureImages]: + final_material_properties = {} + texture_images = {} + + used_material_names = list(material_names) + if not used_material_names and material_properties: + if len(material_properties) > 1: + raise ValueError( + "Multiple materials but no usemtl declarations in the obj file" + ) + # No materials were specified in obj file and only one is in the + # specified .mtl file, so we use it. + used_material_names.append(next(iter(material_properties.keys()))) + + # Only keep the materials referenced in the obj. + for material_name in used_material_names: + if material_name in texture_files: + # Load the texture image. + path = os.path.join(data_dir, texture_files[material_name]) + if path_manager.exists(path): + image = ( + _read_image(path, path_manager=path_manager, format="RGB") / 255.0 + ) + image = torch.from_numpy(image) + texture_images[material_name] = image + else: + msg = f"Texture file does not exist: {path}" + warnings.warn(msg) + + if material_name in material_properties: + final_material_properties[material_name] = material_properties[ + material_name + ] + + return final_material_properties, texture_images + + +def load_mtl( + f: str, + *, + material_names: List[str], + data_dir: str, + device: Device = "cpu", + path_manager: PathManager, +) -> Tuple[MaterialProperties, TextureImages]: + """ + Load texture images and material reflectivity values for ambient, diffuse + and specular light (Ka, Kd, Ks, Ns). + + Args: + f: path to the material information. + material_names: a list of the material names found in the .obj file. + data_dir: the directory where the material texture files are located. + device: Device (as str or torch.tensor) on which to return the new tensors. + path_manager: PathManager for interpreting both f and material_names. + + Returns: + material_properties: dict of properties for each material. If a material + does not have any properties it will have an empty dict. + { + material_name_1: { + "ambient_color": tensor of shape (1, 3), + "diffuse_color": tensor of shape (1, 3), + "specular_color": tensor of shape (1, 3), + "shininess": tensor of shape (1) + }, + material_name_2: {}, + ... + } + texture_images: dict of material names and texture images + { + material_name_1: (H, W, 3) image, + ... + } + """ + material_properties, texture_files = _parse_mtl(f, path_manager, device) + return _load_texture_images( + material_names, + data_dir, + material_properties, + texture_files, + path_manager=path_manager, + ) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/obj_io.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/obj_io.py new file mode 100644 index 0000000000000000000000000000000000000000..156e083022408df152bd21b84f064f146bbf8ca7 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/obj_io.py @@ -0,0 +1,941 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +"""This module implements utility functions for loading and saving meshes.""" +import os +import warnings +from collections import namedtuple +from pathlib import Path +from typing import List, Optional + +import numpy as np +import torch +from iopath.common.file_io import PathManager +from PIL import Image +from pytorch3d.common.datatypes import Device +from pytorch3d.io.mtl_io import load_mtl, make_mesh_texture_atlas +from pytorch3d.io.utils import _check_faces_indices, _make_tensor, _open_file, PathOrStr +from pytorch3d.renderer import TexturesAtlas, TexturesUV +from pytorch3d.structures import join_meshes_as_batch, Meshes + +from .pluggable_formats import endswith, MeshFormatInterpreter + + +# Faces & Aux type returned from load_obj function. +_Faces = namedtuple("Faces", "verts_idx normals_idx textures_idx materials_idx") +_Aux = namedtuple( + "Properties", "normals verts_uvs material_colors texture_images texture_atlas" +) + + +def _format_faces_indices(faces_indices, max_index: int, device, pad_value=None): + """ + Format indices and check for invalid values. Indices can refer to + values in one of the face properties: vertices, textures or normals. + See comments of the load_obj function for more details. + + Args: + faces_indices: List of ints of indices. + max_index: Max index for the face property. + pad_value: if any of the face_indices are padded, specify + the value of the padding (e.g. -1). This is only used + for texture indices indices where there might + not be texture information for all the faces. + + Returns: + faces_indices: List of ints of indices. + + Raises: + ValueError if indices are not in a valid range. + """ + faces_indices = _make_tensor( + faces_indices, cols=3, dtype=torch.int64, device=device + ) + + if pad_value is not None: + mask = faces_indices.eq(pad_value).all(dim=-1) + + # Change to 0 based indexing. + faces_indices[(faces_indices > 0)] -= 1 + + # Negative indexing counts from the end. + faces_indices[(faces_indices < 0)] += max_index + + if pad_value is not None: + # pyre-fixme[61]: `mask` is undefined, or not always defined. + faces_indices[mask] = pad_value + + return _check_faces_indices(faces_indices, max_index, pad_value) + + +def load_obj( + f, + load_textures: bool = True, + create_texture_atlas: bool = False, + texture_atlas_size: int = 4, + texture_wrap: Optional[str] = "repeat", + device: Device = "cpu", + path_manager: Optional[PathManager] = None, +): + """ + Load a mesh from a .obj file and optionally textures from a .mtl file. + Currently this handles verts, faces, vertex texture uv coordinates, normals, + texture images and material reflectivity values. + + Note .obj files are 1-indexed. The tensors returned from this function + are 0-indexed. OBJ spec reference: http://www.martinreddy.net/gfx/3d/OBJ.spec + + Example .obj file format: + :: + # this is a comment + v 1.000000 -1.000000 -1.000000 + v 1.000000 -1.000000 1.000000 + v -1.000000 -1.000000 1.000000 + v -1.000000 -1.000000 -1.000000 + v 1.000000 1.000000 -1.000000 + vt 0.748573 0.750412 + vt 0.749279 0.501284 + vt 0.999110 0.501077 + vt 0.999455 0.750380 + vn 0.000000 0.000000 -1.000000 + vn -1.000000 -0.000000 -0.000000 + vn -0.000000 -0.000000 1.000000 + f 5/2/1 1/2/1 4/3/1 + f 5/1/1 4/3/1 2/4/1 + + The first character of the line denotes the type of input: + :: + - v is a vertex + - vt is the texture coordinate of one vertex + - vn is the normal of one vertex + - f is a face + + Faces are interpreted as follows: + :: + 5/2/1 describes the first vertex of the first triangle + - 5: index of vertex [1.000000 1.000000 -1.000000] + - 2: index of texture coordinate [0.749279 0.501284] + - 1: index of normal [0.000000 0.000000 -1.000000] + + If there are faces with more than 3 vertices + they are subdivided into triangles. Polygonal faces are assumed to have + vertices ordered counter-clockwise so the (right-handed) normal points + out of the screen e.g. a proper rectangular face would be specified like this: + :: + 0_________1 + | | + | | + 3 ________2 + + The face would be split into two triangles: (0, 2, 1) and (0, 3, 2), + both of which are also oriented counter-clockwise and have normals + pointing out of the screen. + + Args: + f: A file-like object (with methods read, readline, tell, and seek), + a pathlib path or a string containing a file name. + load_textures: Boolean indicating whether material files are loaded + create_texture_atlas: Bool, If True a per face texture map is created and + a tensor `texture_atlas` is also returned in `aux`. + texture_atlas_size: Int specifying the resolution of the texture map per face + when `create_texture_atlas=True`. A (texture_size, texture_size, 3) + map is created per face. + texture_wrap: string, one of ["repeat", "clamp"]. This applies when computing + the texture atlas. + If `texture_mode="repeat"`, for uv values outside the range [0, 1] the integer part + is ignored and a repeating pattern is formed. + If `texture_mode="clamp"` the values are clamped to the range [0, 1]. + If None, then there is no transformation of the texture values. + device: Device (as str or torch.device) on which to return the new tensors. + path_manager: optionally a PathManager object to interpret paths. + + Returns: + 6-element tuple containing + + - **verts**: FloatTensor of shape (V, 3). + - **faces**: NamedTuple with fields: + - verts_idx: LongTensor of vertex indices, shape (F, 3). + - normals_idx: (optional) LongTensor of normal indices, shape (F, 3). + - textures_idx: (optional) LongTensor of texture indices, shape (F, 3). + This can be used to index into verts_uvs. + - materials_idx: (optional) List of indices indicating which + material the texture is derived from for each face. + If there is no material for a face, the index is -1. + This can be used to retrieve the corresponding values + in material_colors/texture_images after they have been + converted to tensors or Materials/Textures data + structures - see textures.py and materials.py for + more info. + - **aux**: NamedTuple with fields: + - normals: FloatTensor of shape (N, 3) + - verts_uvs: FloatTensor of shape (T, 2), giving the uv coordinate per + vertex. If a vertex is shared between two faces, it can have + a different uv value for each instance. Therefore it is + possible that the number of verts_uvs is greater than + num verts i.e. T > V. + vertex. + - material_colors: if `load_textures=True` and the material has associated + properties this will be a dict of material names and properties of the form: + + .. code-block:: python + + { + material_name_1: { + "ambient_color": tensor of shape (1, 3), + "diffuse_color": tensor of shape (1, 3), + "specular_color": tensor of shape (1, 3), + "shininess": tensor of shape (1) + }, + material_name_2: {}, + ... + } + + If a material does not have any properties it will have an + empty dict. If `load_textures=False`, `material_colors` will None. + + - texture_images: if `load_textures=True` and the material has a texture map, + this will be a dict of the form: + + .. code-block:: python + + { + material_name_1: (H, W, 3) image, + ... + } + If `load_textures=False`, `texture_images` will None. + - texture_atlas: if `load_textures=True` and `create_texture_atlas=True`, + this will be a FloatTensor of the form: (F, texture_size, textures_size, 3) + If the material does not have a texture map, then all faces + will have a uniform white texture. Otherwise `texture_atlas` will be + None. + """ + data_dir = "./" + if isinstance(f, (str, bytes, Path)): + # pyre-fixme[6]: For 1st argument expected `PathLike[Variable[AnyStr <: + # [str, bytes]]]` but got `Union[Path, bytes, str]`. + data_dir = os.path.dirname(f) + if path_manager is None: + path_manager = PathManager() + with _open_file(f, path_manager, "r") as f: + return _load_obj( + f, + data_dir=data_dir, + load_textures=load_textures, + create_texture_atlas=create_texture_atlas, + texture_atlas_size=texture_atlas_size, + texture_wrap=texture_wrap, + path_manager=path_manager, + device=device, + ) + + +def load_objs_as_meshes( + files: list, + device: Optional[Device] = None, + load_textures: bool = True, + create_texture_atlas: bool = False, + texture_atlas_size: int = 4, + texture_wrap: Optional[str] = "repeat", + path_manager: Optional[PathManager] = None, +): + """ + Load meshes from a list of .obj files using the load_obj function, and + return them as a Meshes object. This only works for meshes which have a + single texture image for the whole mesh. See the load_obj function for more + details. material_colors and normals are not stored. + + Args: + files: A list of file-like objects (with methods read, readline, tell, + and seek), pathlib paths or strings containing file names. + device: Desired device of returned Meshes. Default: + uses the current device for the default tensor type. + load_textures: Boolean indicating whether material files are loaded + create_texture_atlas, texture_atlas_size, texture_wrap: as for load_obj. + path_manager: optionally a PathManager object to interpret paths. + + Returns: + New Meshes object. + """ + mesh_list = [] + for f_obj in files: + verts, faces, aux = load_obj( + f_obj, + load_textures=load_textures, + create_texture_atlas=create_texture_atlas, + texture_atlas_size=texture_atlas_size, + texture_wrap=texture_wrap, + path_manager=path_manager, + ) + tex = None + if create_texture_atlas: + # TexturesAtlas type + tex = TexturesAtlas(atlas=[aux.texture_atlas.to(device)]) + else: + # TexturesUV type + tex_maps = aux.texture_images + if tex_maps is not None and len(tex_maps) > 0: + verts_uvs = aux.verts_uvs.to(device) # (V, 2) + faces_uvs = faces.textures_idx.to(device) # (F, 3) + image = list(tex_maps.values())[0].to(device)[None] + tex = TexturesUV( + verts_uvs=[verts_uvs], faces_uvs=[faces_uvs], maps=image + ) + + mesh = Meshes( + verts=[verts.to(device)], faces=[faces.verts_idx.to(device)], textures=tex + ) + mesh_list.append(mesh) + if len(mesh_list) == 1: + return mesh_list[0] + return join_meshes_as_batch(mesh_list) + + +class MeshObjFormat(MeshFormatInterpreter): + def __init__(self) -> None: + self.known_suffixes = (".obj",) + + def read( + self, + path: PathOrStr, + include_textures: bool, + device: Device, + path_manager: PathManager, + create_texture_atlas: bool = False, + texture_atlas_size: int = 4, + texture_wrap: Optional[str] = "repeat", + **kwargs, + ) -> Optional[Meshes]: + if not endswith(path, self.known_suffixes): + return None + mesh = load_objs_as_meshes( + files=[path], + device=device, + load_textures=include_textures, + create_texture_atlas=create_texture_atlas, + texture_atlas_size=texture_atlas_size, + texture_wrap=texture_wrap, + path_manager=path_manager, + ) + return mesh + + def save( + self, + data: Meshes, + path: PathOrStr, + path_manager: PathManager, + binary: Optional[bool], + decimal_places: Optional[int] = None, + **kwargs, + ) -> bool: + if not endswith(path, self.known_suffixes): + return False + + verts = data.verts_list()[0] + faces = data.faces_list()[0] + + verts_uvs: Optional[torch.Tensor] = None + faces_uvs: Optional[torch.Tensor] = None + texture_map: Optional[torch.Tensor] = None + + if isinstance(data.textures, TexturesUV): + verts_uvs = data.textures.verts_uvs_padded()[0] + faces_uvs = data.textures.faces_uvs_padded()[0] + texture_map = data.textures.maps_padded()[0] + + save_obj( + f=path, + verts=verts, + faces=faces, + decimal_places=decimal_places, + path_manager=path_manager, + verts_uvs=verts_uvs, + faces_uvs=faces_uvs, + texture_map=texture_map, + ) + return True + + +def _parse_face( + line, + tokens, + material_idx, + faces_verts_idx, + faces_normals_idx, + faces_textures_idx, + faces_materials_idx, +) -> None: + face = tokens[1:] + face_list = [f.split("/") for f in face] + face_verts = [] + face_normals = [] + face_textures = [] + + for vert_props in face_list: + # Vertex index. + face_verts.append(int(vert_props[0])) + if len(vert_props) > 1: + if vert_props[1] != "": + # Texture index is present e.g. f 4/1/1. + face_textures.append(int(vert_props[1])) + if len(vert_props) > 2: + # Normal index present e.g. 4/1/1 or 4//1. + face_normals.append(int(vert_props[2])) + if len(vert_props) > 3: + raise ValueError( + "Face vertices can only have 3 properties. \ + Face vert %s, Line: %s" + % (str(vert_props), str(line)) + ) + + # Triplets must be consistent for all vertices in a face e.g. + # legal statement: f 4/1/1 3/2/1 2/1/1. + # illegal statement: f 4/1/1 3//1 2//1. + # If the face does not have normals or textures indices + # fill with pad value = -1. This will ensure that + # all the face index tensors will have F values where + # F is the number of faces. + if len(face_normals) > 0: + if not (len(face_verts) == len(face_normals)): + raise ValueError( + "Face %s is an illegal statement. \ + Vertex properties are inconsistent. Line: %s" + % (str(face), str(line)) + ) + else: + face_normals = [-1] * len(face_verts) # Fill with -1 + if len(face_textures) > 0: + if not (len(face_verts) == len(face_textures)): + raise ValueError( + "Face %s is an illegal statement. \ + Vertex properties are inconsistent. Line: %s" + % (str(face), str(line)) + ) + else: + face_textures = [-1] * len(face_verts) # Fill with -1 + + # Subdivide faces with more than 3 vertices. + # See comments of the load_obj function for more details. + for i in range(len(face_verts) - 2): + faces_verts_idx.append((face_verts[0], face_verts[i + 1], face_verts[i + 2])) + faces_normals_idx.append( + (face_normals[0], face_normals[i + 1], face_normals[i + 2]) + ) + faces_textures_idx.append( + (face_textures[0], face_textures[i + 1], face_textures[i + 2]) + ) + faces_materials_idx.append(material_idx) + + +def _parse_obj(f, data_dir: str): + """ + Load a mesh from a file-like object. See load_obj function for more details + about the return values. + """ + verts, normals, verts_uvs = [], [], [] + faces_verts_idx, faces_normals_idx, faces_textures_idx = [], [], [] + faces_materials_idx = [] + material_names = [] + mtl_path = None + + lines = [line.strip() for line in f] + + # startswith expects each line to be a string. If the file is read in as + # bytes then first decode to strings. + if lines and isinstance(lines[0], bytes): + lines = [el.decode("utf-8") for el in lines] + + materials_idx = -1 + + for line in lines: + tokens = line.strip().split() + if line.startswith("mtllib"): + if len(tokens) < 2: + raise ValueError("material file name is not specified") + # NOTE: only allow one .mtl file per .obj. + # Definitions for multiple materials can be included + # in this one .mtl file. + mtl_path = line[len(tokens[0]) :].strip() # Take the remainder of the line + mtl_path = os.path.join(data_dir, mtl_path) + elif len(tokens) and tokens[0] == "usemtl": + material_name = tokens[1] + # materials are often repeated for different parts + # of a mesh. + if material_name not in material_names: + material_names.append(material_name) + materials_idx = len(material_names) - 1 + else: + materials_idx = material_names.index(material_name) + elif line.startswith("v "): # Line is a vertex. + vert = [float(x) for x in tokens[1:4]] + if len(vert) != 3: + msg = "Vertex %s does not have 3 values. Line: %s" + raise ValueError(msg % (str(vert), str(line))) + verts.append(vert) + elif line.startswith("vt "): # Line is a texture. + tx = [float(x) for x in tokens[1:3]] + if len(tx) != 2: + raise ValueError( + "Texture %s does not have 2 values. Line: %s" % (str(tx), str(line)) + ) + verts_uvs.append(tx) + elif line.startswith("vn "): # Line is a normal. + norm = [float(x) for x in tokens[1:4]] + if len(norm) != 3: + msg = "Normal %s does not have 3 values. Line: %s" + raise ValueError(msg % (str(norm), str(line))) + normals.append(norm) + elif line.startswith("f "): # Line is a face. + # Update face properties info. + _parse_face( + line, + tokens, + materials_idx, + faces_verts_idx, + faces_normals_idx, + faces_textures_idx, + faces_materials_idx, + ) + + return ( + verts, + normals, + verts_uvs, + faces_verts_idx, + faces_normals_idx, + faces_textures_idx, + faces_materials_idx, + material_names, + mtl_path, + ) + + +def _load_materials( + material_names: List[str], + f: Optional[str], + *, + data_dir: str, + load_textures: bool, + device: Device, + path_manager: PathManager, +): + """ + Load materials and optionally textures from the specified path. + + Args: + material_names: a list of the material names found in the .obj file. + f: path to the material information. + data_dir: the directory where the material texture files are located. + load_textures: whether textures should be loaded. + device: Device (as str or torch.device) on which to return the new tensors. + path_manager: PathManager object to interpret paths. + + Returns: + material_colors: dict of properties for each material. + texture_images: dict of material names and texture images. + """ + if not load_textures: + return None, None + + if f is None: + warnings.warn("No mtl file provided") + return None, None + + if not path_manager.exists(f): + warnings.warn(f"Mtl file does not exist: {f}") + return None, None + + # Texture mode uv wrap + return load_mtl( + f, + material_names=material_names, + data_dir=data_dir, + path_manager=path_manager, + device=device, + ) + + +def _load_obj( + f_obj, + *, + data_dir: str, + load_textures: bool = True, + create_texture_atlas: bool = False, + texture_atlas_size: int = 4, + texture_wrap: Optional[str] = "repeat", + path_manager: PathManager, + device: Device = "cpu", +): + """ + Load a mesh from a file-like object. See load_obj function more details. + Any material files associated with the obj are expected to be in the + directory given by data_dir. + """ + + if texture_wrap is not None and texture_wrap not in ["repeat", "clamp"]: + msg = "texture_wrap must be one of ['repeat', 'clamp'] or None, got %s" + raise ValueError(msg % texture_wrap) + + ( + verts, + normals, + verts_uvs, + faces_verts_idx, + faces_normals_idx, + faces_textures_idx, + faces_materials_idx, + material_names, + mtl_path, + ) = _parse_obj(f_obj, data_dir) + + verts = _make_tensor(verts, cols=3, dtype=torch.float32, device=device) # (V, 3) + normals = _make_tensor( + normals, + cols=3, + dtype=torch.float32, + device=device, + ) # (N, 3) + verts_uvs = _make_tensor( + verts_uvs, + cols=2, + dtype=torch.float32, + device=device, + ) # (T, 2) + + faces_verts_idx = _format_faces_indices( + faces_verts_idx, verts.shape[0], device=device + ) + + # Repeat for normals and textures if present. + if len(faces_normals_idx): + faces_normals_idx = _format_faces_indices( + faces_normals_idx, normals.shape[0], device=device, pad_value=-1 + ) + if len(faces_textures_idx): + faces_textures_idx = _format_faces_indices( + faces_textures_idx, verts_uvs.shape[0], device=device, pad_value=-1 + ) + if len(faces_materials_idx): + faces_materials_idx = torch.tensor( + faces_materials_idx, dtype=torch.int64, device=device + ) + + texture_atlas = None + material_colors, texture_images = _load_materials( + material_names, + mtl_path, + data_dir=data_dir, + load_textures=load_textures, + path_manager=path_manager, + device=device, + ) + + if material_colors and not material_names: + # usemtl was not present but single material was present in the .mtl file + material_names.append(next(iter(material_colors.keys()))) + # replace all -1 by 0 material idx + if torch.is_tensor(faces_materials_idx): + faces_materials_idx.clamp_(min=0) + + if create_texture_atlas: + # Using the images and properties from the + # material file make a per face texture map. + + # Create an array of strings of material names for each face. + # If faces_materials_idx == -1 then that face doesn't have a material. + idx = faces_materials_idx.cpu().numpy() + face_material_names = np.array(material_names)[idx] # (F,) + face_material_names[idx == -1] = "" + + # Construct the atlas. + texture_atlas = make_mesh_texture_atlas( + material_colors, + texture_images, + face_material_names, + faces_textures_idx, + verts_uvs, + texture_atlas_size, + texture_wrap, + ) + + faces = _Faces( + verts_idx=faces_verts_idx, + normals_idx=faces_normals_idx, + textures_idx=faces_textures_idx, + materials_idx=faces_materials_idx, + ) + aux = _Aux( + normals=normals if len(normals) else None, + verts_uvs=verts_uvs if len(verts_uvs) else None, + material_colors=material_colors, + texture_images=texture_images, + texture_atlas=texture_atlas, + ) + return verts, faces, aux + + +def save_obj( + f: PathOrStr, + verts, + faces, + decimal_places: Optional[int] = None, + path_manager: Optional[PathManager] = None, + *, + normals: Optional[torch.Tensor] = None, + faces_normals_idx: Optional[torch.Tensor] = None, + verts_uvs: Optional[torch.Tensor] = None, + faces_uvs: Optional[torch.Tensor] = None, + texture_map: Optional[torch.Tensor] = None, +) -> None: + """ + Save a mesh to an .obj file. + + Args: + f: File (str or path) to which the mesh should be written. + verts: FloatTensor of shape (V, 3) giving vertex coordinates. + faces: LongTensor of shape (F, 3) giving faces. + decimal_places: Number of decimal places for saving. + path_manager: Optional PathManager for interpreting f if + it is a str. + normals: FloatTensor of shape (V, 3) giving normals for faces_normals_idx + to index into. + faces_normals_idx: LongTensor of shape (F, 3) giving the index into + normals for each vertex in the face. + verts_uvs: FloatTensor of shape (V, 2) giving the uv coordinate per vertex. + faces_uvs: LongTensor of shape (F, 3) giving the index into verts_uvs for + each vertex in the face. + texture_map: FloatTensor of shape (H, W, 3) representing the texture map + for the mesh which will be saved as an image. The values are expected + to be in the range [0, 1], + """ + if len(verts) and (verts.dim() != 2 or verts.size(1) != 3): + message = "'verts' should either be empty or of shape (num_verts, 3)." + raise ValueError(message) + + if len(faces) and (faces.dim() != 2 or faces.size(1) != 3): + message = "'faces' should either be empty or of shape (num_faces, 3)." + raise ValueError(message) + + if (normals is None) != (faces_normals_idx is None): + message = "'normals' and 'faces_normals_idx' must both be None or neither." + raise ValueError(message) + + if faces_normals_idx is not None and ( + faces_normals_idx.dim() != 2 or faces_normals_idx.size(1) != 3 + ): + message = ( + "'faces_normals_idx' should either be empty or of shape (num_faces, 3)." + ) + raise ValueError(message) + + if normals is not None and (normals.dim() != 2 or normals.size(1) != 3): + message = "'normals' should either be empty or of shape (num_verts, 3)." + raise ValueError(message) + + if faces_uvs is not None and (faces_uvs.dim() != 2 or faces_uvs.size(1) != 3): + message = "'faces_uvs' should either be empty or of shape (num_faces, 3)." + raise ValueError(message) + + if verts_uvs is not None and (verts_uvs.dim() != 2 or verts_uvs.size(1) != 2): + message = "'verts_uvs' should either be empty or of shape (num_verts, 2)." + raise ValueError(message) + + if texture_map is not None and (texture_map.dim() != 3 or texture_map.size(2) != 3): + message = "'texture_map' should either be empty or of shape (H, W, 3)." + raise ValueError(message) + + if path_manager is None: + path_manager = PathManager() + + save_texture = all(t is not None for t in [faces_uvs, verts_uvs, texture_map]) + output_path = Path(f) + + # Save the .obj file + with _open_file(f, path_manager, "w") as f: + if save_texture: + # Add the header required for the texture info to be loaded correctly + obj_header = "\nmtllib {0}.mtl\nusemtl mesh\n\n".format(output_path.stem) + f.write(obj_header) + _save( + f, + verts, + faces, + decimal_places, + normals=normals, + faces_normals_idx=faces_normals_idx, + verts_uvs=verts_uvs, + faces_uvs=faces_uvs, + save_texture=save_texture, + save_normals=normals is not None, + ) + + # Save the .mtl and .png files associated with the texture + if save_texture: + image_path = output_path.with_suffix(".png") + mtl_path = output_path.with_suffix(".mtl") + if isinstance(f, str): + # Back to str for iopath interpretation. + image_path = str(image_path) + mtl_path = str(mtl_path) + + # Save texture map to output folder + # pyre-fixme[16] # undefined attribute cpu + texture_map = texture_map.detach().cpu() * 255.0 + image = Image.fromarray(texture_map.numpy().astype(np.uint8)) + with _open_file(image_path, path_manager, "wb") as im_f: + image.save(im_f) + + # Create .mtl file with the material name and texture map filename + # TODO: enable material properties to also be saved. + with _open_file(mtl_path, path_manager, "w") as f_mtl: + lines = f"newmtl mesh\n" f"map_Kd {output_path.stem}.png\n" + f_mtl.write(lines) + + +# TODO (nikhilar) Speed up this function. +def _save( + f, + verts, + faces, + decimal_places: Optional[int] = None, + *, + normals: Optional[torch.Tensor] = None, + faces_normals_idx: Optional[torch.Tensor] = None, + verts_uvs: Optional[torch.Tensor] = None, + faces_uvs: Optional[torch.Tensor] = None, + save_texture: bool = False, + save_normals: bool = False, +) -> None: + + if len(verts) and (verts.dim() != 2 or verts.size(1) != 3): + message = "'verts' should either be empty or of shape (num_verts, 3)." + raise ValueError(message) + + if len(faces) and (faces.dim() != 2 or faces.size(1) != 3): + message = "'faces' should either be empty or of shape (num_faces, 3)." + raise ValueError(message) + + if not (len(verts) or len(faces)): + warnings.warn("Empty 'verts' and 'faces' arguments provided") + return + + verts, faces = verts.cpu(), faces.cpu() + + lines = "" + + if decimal_places is None: + float_str = "%f" + else: + float_str = "%" + ".%df" % decimal_places + + if len(verts): + V, D = verts.shape + for i in range(V): + vert = [float_str % verts[i, j] for j in range(D)] + lines += "v %s\n" % " ".join(vert) + + if save_normals: + assert normals is not None + assert faces_normals_idx is not None + lines += _write_normals(normals, faces_normals_idx, float_str) + + if save_texture: + assert faces_uvs is not None + assert verts_uvs is not None + + if faces_uvs is not None and (faces_uvs.dim() != 2 or faces_uvs.size(1) != 3): + message = "'faces_uvs' should either be empty or of shape (num_faces, 3)." + raise ValueError(message) + + if verts_uvs is not None and (verts_uvs.dim() != 2 or verts_uvs.size(1) != 2): + message = "'verts_uvs' should either be empty or of shape (num_verts, 2)." + raise ValueError(message) + + verts_uvs, faces_uvs = verts_uvs.cpu(), faces_uvs.cpu() + + # Save verts uvs after verts + if len(verts_uvs): + uV, uD = verts_uvs.shape + for i in range(uV): + uv = [float_str % verts_uvs[i, j] for j in range(uD)] + lines += "vt %s\n" % " ".join(uv) + + f.write(lines) + + if torch.any(faces >= verts.shape[0]) or torch.any(faces < 0): + warnings.warn("Faces have invalid indices") + + if len(faces): + _write_faces( + f, + faces, + faces_uvs if save_texture else None, + faces_normals_idx if save_normals else None, + ) + + +def _write_normals( + normals: torch.Tensor, faces_normals_idx: torch.Tensor, float_str: str +) -> str: + if faces_normals_idx.dim() != 2 or faces_normals_idx.size(1) != 3: + message = ( + "'faces_normals_idx' should either be empty or of shape (num_faces, 3)." + ) + raise ValueError(message) + + if normals.dim() != 2 or normals.size(1) != 3: + message = "'normals' should either be empty or of shape (num_verts, 3)." + raise ValueError(message) + + normals, faces_normals_idx = normals.cpu(), faces_normals_idx.cpu() + + lines = [] + V, D = normals.shape + for i in range(V): + normal = [float_str % normals[i, j] for j in range(D)] + lines.append("vn %s\n" % " ".join(normal)) + return "".join(lines) + + +def _write_faces( + f, + faces: torch.Tensor, + faces_uvs: Optional[torch.Tensor], + faces_normals_idx: Optional[torch.Tensor], +) -> None: + F, P = faces.shape + for i in range(F): + if faces_normals_idx is not None: + if faces_uvs is not None: + # Format faces as {verts_idx}/{verts_uvs_idx}/{verts_normals_idx} + face = [ + "%d/%d/%d" + % ( + faces[i, j] + 1, + faces_uvs[i, j] + 1, + faces_normals_idx[i, j] + 1, + ) + for j in range(P) + ] + else: + # Format faces as {verts_idx}//{verts_normals_idx} + face = [ + "%d//%d" % (faces[i, j] + 1, faces_normals_idx[i, j] + 1) + for j in range(P) + ] + elif faces_uvs is not None: + # Format faces as {verts_idx}/{verts_uvs_idx} + face = ["%d/%d" % (faces[i, j] + 1, faces_uvs[i, j] + 1) for j in range(P)] + else: + face = ["%d" % (faces[i, j] + 1) for j in range(P)] + + if i + 1 < F: + f.write("f %s\n" % " ".join(face)) + else: + # No newline at the end of the file. + f.write("f %s" % " ".join(face)) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/off_io.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/off_io.py new file mode 100644 index 0000000000000000000000000000000000000000..78b4390d493d0d16bff73655d566e5cbbc0c657f --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/off_io.py @@ -0,0 +1,496 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +""" +This module implements utility functions for loading and saving +meshes as .off files. + +This format is introduced, for example, at +http://www.geomview.org/docs/html/OFF.html . +""" +import warnings +from typing import cast, Optional, Tuple, Union + +import numpy as np +import torch +from iopath.common.file_io import PathManager +from pytorch3d.io.utils import _check_faces_indices, _open_file, PathOrStr +from pytorch3d.renderer import TexturesAtlas, TexturesVertex +from pytorch3d.structures import Meshes + +from .pluggable_formats import endswith, MeshFormatInterpreter + + +def _is_line_empty(line: Union[str, bytes]) -> bool: + """ + Returns whether line is not relevant in an OFF file. + """ + line = line.strip() + return len(line) == 0 or line[:1] == b"#" + + +def _count_next_line_periods(file) -> int: + """ + Returns the number of . characters before any # on the next + meaningful line. + """ + old_offset = file.tell() + line = file.readline() + while _is_line_empty(line): + line = file.readline() + if len(line) == 0: + raise ValueError("Premature end of file") + + contents = line.split(b"#")[0] + count = contents.count(b".") + file.seek(old_offset) + return count + + +def _read_faces_lump( + file, n_faces: int, n_colors: Optional[int] +) -> Optional[Tuple[np.ndarray, int, Optional[np.ndarray]]]: + """ + Parse n_faces faces and faces_colors from the file, + if they all have the same number of vertices. + This is used in two ways. + 1) To try to read all faces. + 2) To read faces one-by-one if that failed. + + Args: + file: file-like object being read. + n_faces: The known number of faces yet to read. + n_colors: The number of colors if known already. + + Returns: + - 2D numpy array of faces + - number of colors found + - 2D numpy array of face colors if found. + of None if there are faces with different numbers of vertices. + """ + if n_faces == 0: + return np.array([[]]), 0, None + old_offset = file.tell() + try: + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", message=".* Empty input file.*", category=UserWarning + ) + data = np.loadtxt(file, dtype=np.float32, ndmin=2, max_rows=n_faces) + except ValueError as e: + if n_faces > 1 and "Wrong number of columns" in e.args[0]: + file.seek(old_offset) + return None + raise ValueError("Not enough face data.") from None + + if len(data) != n_faces: + raise ValueError("Not enough face data.") + face_size = int(data[0, 0]) + if (data[:, 0] != face_size).any(): + msg = "A line of face data did not have the specified length." + raise ValueError(msg) + if face_size < 3: + raise ValueError("Faces must have at least 3 vertices.") + + n_colors_found = data.shape[1] - 1 - face_size + if n_colors is not None and n_colors_found != n_colors: + raise ValueError("Number of colors differs between faces.") + n_colors = n_colors_found + if n_colors not in [0, 3, 4]: + raise ValueError("Unexpected number of colors.") + + face_raw_data = data[:, 1 : 1 + face_size].astype("int64") + if face_size == 3: + face_data = face_raw_data + else: + face_arrays = [ + face_raw_data[:, [0, i + 1, i + 2]] for i in range(face_size - 2) + ] + face_data = np.vstack(face_arrays) + + if n_colors == 0: + return face_data, 0, None + colors = data[:, 1 + face_size :] + if face_size == 3: + return face_data, n_colors, colors + return face_data, n_colors, np.tile(colors, (face_size - 2, 1)) + + +def _read_faces( + file, n_faces: int +) -> Tuple[Optional[np.ndarray], Optional[np.ndarray]]: + """ + Returns faces and face colors from the file. + + Args: + file: file-like object being read. + n_faces: The known number of faces. + + Returns: + 2D numpy arrays of faces and face colors, or None for each if + they are not present. + """ + if n_faces == 0: + return None, None + + color_is_int = 0 == _count_next_line_periods(file) + color_scale = 1 / 255.0 if color_is_int else 1 + + faces_ncolors_colors = _read_faces_lump(file, n_faces=n_faces, n_colors=None) + if faces_ncolors_colors is not None: + faces, _, colors = faces_ncolors_colors + if colors is None: + return faces, None + return faces, colors * color_scale + + faces_list, colors_list = [], [] + n_colors = None + for _ in range(n_faces): + faces_ncolors_colors = _read_faces_lump(file, n_faces=1, n_colors=n_colors) + faces_found, n_colors, colors_found = cast( + Tuple[np.ndarray, int, Optional[np.ndarray]], faces_ncolors_colors + ) + faces_list.append(faces_found) + colors_list.append(colors_found) + faces = np.vstack(faces_list) + if n_colors == 0: + colors = None + else: + colors = np.vstack(colors_list) * color_scale + return faces, colors + + +def _read_verts(file, n_verts: int) -> Tuple[np.ndarray, Optional[np.ndarray]]: + """ + Returns verts and vertex colors from the file. + + Args: + file: file-like object being read. + n_verts: The known number of faces. + + Returns: + 2D numpy arrays of verts and (if present) + vertex colors. + """ + + color_is_int = 3 == _count_next_line_periods(file) + color_scale = 1 / 255.0 if color_is_int else 1 + + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", message=".* Empty input file.*", category=UserWarning + ) + data = np.loadtxt(file, dtype=np.float32, ndmin=2, max_rows=n_verts) + if data.shape[0] != n_verts: + raise ValueError("Not enough vertex data.") + if data.shape[1] not in [3, 6, 7]: + raise ValueError("Bad vertex data.") + + if data.shape[1] == 3: + return data, None + return data[:, :3], data[:, 3:] * color_scale # [] + + +def _load_off_stream(file) -> dict: + """ + Load the data from a stream of an .off file. + + Example .off file format: + + off + 8 6 1927 { number of vertices, faces, and (not used) edges } + # comment { comments with # sign } + 0 0 0 { start of vertex list } + 0 0 1 + 0 1 1 + 0 1 0 + 1 0 0 + 1 0 1 + 1 1 1 + 1 1 0 + 4 0 1 2 3 { start of face list } + 4 7 6 5 4 + 4 0 4 5 1 + 4 1 5 6 2 + 4 2 6 7 3 + 4 3 7 4 0 + + Args: + file: A binary file-like object (with methods read, readline, + tell and seek). + + Returns dictionary possibly containing: + verts: (always present) FloatTensor of shape (V, 3). + verts_colors: FloatTensor of shape (V, C) where C is 3 or 4. + faces: LongTensor of vertex indices, split into triangles, shape (F, 3). + faces_colors: FloatTensor of shape (F, C), where C is 3 or 4. + """ + header = file.readline() + + while _is_line_empty(header): + header = file.readline() + + if header[:3].lower() == b"off": + header = header[3:] + + while _is_line_empty(header): + header = file.readline() + + items = header.split() + if len(items) < 3: + raise ValueError("Invalid counts line: %s" % header) + + try: + n_verts = int(items[0]) + except ValueError: + raise ValueError("Invalid counts line: %s" % header) from None + try: + n_faces = int(items[1]) + except ValueError: + raise ValueError("Invalid counts line: %s" % header) from None + + if (len(items) > 3 and not items[3].startswith(b"#")) or n_verts < 0 or n_faces < 0: + raise ValueError("Invalid counts line: %s" % header) + + verts, verts_colors = _read_verts(file, n_verts) + faces, faces_colors = _read_faces(file, n_faces) + + end = file.read().strip() + if len(end) != 0: + raise ValueError("Extra data at end of file: " + str(end[:20])) + + out = {"verts": verts} + if verts_colors is not None: + out["verts_colors"] = verts_colors + if faces is not None: + out["faces"] = faces + if faces_colors is not None: + out["faces_colors"] = faces_colors + return out + + +def _write_off_data( + file, + verts: torch.Tensor, + verts_colors: Optional[torch.Tensor] = None, + faces: Optional[torch.LongTensor] = None, + faces_colors: Optional[torch.Tensor] = None, + decimal_places: Optional[int] = None, +) -> None: + """ + Internal implementation for saving 3D data to a .off file. + + Args: + file: Binary file object to which the 3D data should be written. + verts: FloatTensor of shape (V, 3) giving vertex coordinates. + verts_colors: FloatTensor of shape (V, C) giving vertex colors where C is 3 or 4. + faces: LongTensor of shape (F, 3) giving faces. + faces_colors: FloatTensor of shape (V, C) giving face colors where C is 3 or 4. + decimal_places: Number of decimal places for saving. + """ + nfaces = 0 if faces is None else faces.shape[0] + file.write(f"off\n{verts.shape[0]} {nfaces} 0\n".encode("ascii")) + + if verts_colors is not None: + verts = torch.cat((verts, verts_colors), dim=1) + if decimal_places is None: + float_str = "%f" + else: + float_str = "%" + ".%df" % decimal_places + np.savetxt(file, verts.cpu().detach().numpy(), float_str) + + if faces is not None: + _check_faces_indices(faces, max_index=verts.shape[0]) + + if faces_colors is not None: + face_data = torch.cat( + [ + cast(torch.Tensor, faces).cpu().to(torch.float64), + faces_colors.detach().cpu().to(torch.float64), + ], + dim=1, + ) + format = "3 %d %d %d" + " %f" * faces_colors.shape[1] + np.savetxt(file, face_data.numpy(), format) + elif faces is not None: + np.savetxt(file, faces.cpu().detach().numpy(), "3 %d %d %d") + + +def _save_off( + file, + *, + verts: torch.Tensor, + verts_colors: Optional[torch.Tensor] = None, + faces: Optional[torch.LongTensor] = None, + faces_colors: Optional[torch.Tensor] = None, + decimal_places: Optional[int] = None, + path_manager: PathManager, +) -> None: + """ + Save a mesh to an ascii .off file. + + Args: + file: File (or path) to which the mesh should be written. + verts: FloatTensor of shape (V, 3) giving vertex coordinates. + verts_colors: FloatTensor of shape (V, C) giving vertex colors where C is 3 or 4. + faces: LongTensor of shape (F, 3) giving faces. + faces_colors: FloatTensor of shape (V, C) giving face colors where C is 3 or 4. + decimal_places: Number of decimal places for saving. + """ + if len(verts) and not (verts.dim() == 2 and verts.size(1) == 3): + message = "Argument 'verts' should either be empty or of shape (num_verts, 3)." + raise ValueError(message) + + if verts_colors is not None and 0 == len(verts_colors): + verts_colors = None + if faces_colors is not None and 0 == len(faces_colors): + faces_colors = None + if faces is not None and 0 == len(faces): + faces = None + + if verts_colors is not None: + if not (verts_colors.dim() == 2 and verts_colors.size(1) in [3, 4]): + message = "verts_colors should have shape (num_faces, C)." + raise ValueError(message) + if verts_colors.shape[0] != verts.shape[0]: + message = "verts_colors should have the same length as verts." + raise ValueError(message) + + if faces is not None and not (faces.dim() == 2 and faces.size(1) == 3): + message = "Argument 'faces' if present should have shape (num_faces, 3)." + raise ValueError(message) + if faces_colors is not None and faces is None: + message = "Cannot have face colors without faces" + raise ValueError(message) + + if faces_colors is not None: + if not (faces_colors.dim() == 2 and faces_colors.size(1) in [3, 4]): + message = "faces_colors should have shape (num_faces, C)." + raise ValueError(message) + if faces_colors.shape[0] != cast(torch.LongTensor, faces).shape[0]: + message = "faces_colors should have the same length as faces." + raise ValueError(message) + + with _open_file(file, path_manager, "wb") as f: + _write_off_data(f, verts, verts_colors, faces, faces_colors, decimal_places) + + +class MeshOffFormat(MeshFormatInterpreter): + """ + Loads and saves meshes in the ascii OFF format. This is a simple + format which can only deal with the following texture types: + + - TexturesVertex, i.e. one color for each vertex + - TexturesAtlas with R=1, i.e. one color for each face. + + There are some possible features of OFF files which we do not support + and which appear to be rare: + + - Four dimensional data. + - Binary data. + - Vertex Normals. + - Texture coordinates. + - "COFF" header. + + Example .off file format: + + off + 8 6 1927 { number of vertices, faces, and (not used) edges } + # comment { comments with # sign } + 0 0 0 { start of vertex list } + 0 0 1 + 0 1 1 + 0 1 0 + 1 0 0 + 1 0 1 + 1 1 1 + 1 1 0 + 4 0 1 2 3 { start of face list } + 4 7 6 5 4 + 4 0 4 5 1 + 4 1 5 6 2 + 4 2 6 7 3 + 4 3 7 4 0 + + """ + + def __init__(self) -> None: + self.known_suffixes = (".off",) + + def read( + self, + path: PathOrStr, + include_textures: bool, + device, + path_manager: PathManager, + **kwargs, + ) -> Optional[Meshes]: + if not endswith(path, self.known_suffixes): + return None + + with _open_file(path, path_manager, "rb") as f: + data = _load_off_stream(f) + verts = torch.from_numpy(data["verts"]).to(device) + if "faces" in data: + faces = torch.from_numpy(data["faces"]).to(dtype=torch.int64, device=device) + else: + faces = torch.zeros((0, 3), dtype=torch.int64, device=device) + + textures = None + if "verts_colors" in data: + if "faces_colors" in data: + msg = "Faces colors ignored because vertex colors provided too." + warnings.warn(msg) + verts_colors = torch.from_numpy(data["verts_colors"]).to(device) + textures = TexturesVertex([verts_colors]) + elif "faces_colors" in data: + faces_colors = torch.from_numpy(data["faces_colors"]).to(device) + textures = TexturesAtlas([faces_colors[:, None, None, :]]) + + mesh = Meshes( + verts=[verts.to(device)], faces=[faces.to(device)], textures=textures + ) + return mesh + + def save( + self, + data: Meshes, + path: PathOrStr, + path_manager: PathManager, + binary: Optional[bool], + decimal_places: Optional[int] = None, + **kwargs, + ) -> bool: + if not endswith(path, self.known_suffixes): + return False + + verts = data.verts_list()[0] + faces = data.faces_list()[0] + if isinstance(data.textures, TexturesVertex): + [verts_colors] = data.textures.verts_features_list() + else: + verts_colors = None + + faces_colors = None + if isinstance(data.textures, TexturesAtlas): + [atlas] = data.textures.atlas_list() + F, R, _, D = atlas.shape + if R == 1: + faces_colors = atlas[:, 0, 0, :] + + _save_off( + file=path, + verts=verts, + faces=faces, + verts_colors=verts_colors, + faces_colors=faces_colors, + decimal_places=decimal_places, + path_manager=path_manager, + ) + return True diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/pluggable.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/pluggable.py new file mode 100644 index 0000000000000000000000000000000000000000..4e9951ac5f29296dfd059d5632b9d2a89c091013 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/pluggable.py @@ -0,0 +1,225 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +from collections import deque +from pathlib import Path +from typing import Deque, Optional, Union + +from iopath.common.file_io import PathManager +from pytorch3d.common.datatypes import Device +from pytorch3d.structures import Meshes, Pointclouds + +from .obj_io import MeshObjFormat +from .off_io import MeshOffFormat +from .pluggable_formats import MeshFormatInterpreter, PointcloudFormatInterpreter +from .ply_io import MeshPlyFormat, PointcloudPlyFormat + + +""" +This module has the master functions for loading and saving data. + +The main usage is via the IO object, and its methods +`load_mesh`, `save_mesh`, `load_pointcloud` and `save_pointcloud`. + +For example, to load a mesh you might do:: + + from pytorch3d.io import IO + + mesh = IO().load_mesh("mymesh.obj") + +and to save a point cloud you might do:: + + pcl = Pointclouds(...) + IO().save_pointcloud(pcl, "output_pointcloud.obj") + +""" + + +class IO: + """ + This class is the interface to flexible loading and saving of meshes and point clouds. + + In simple cases the user will just initialize an instance of this class as `IO()` + and then use its load and save functions. The arguments of the initializer are not + usually needed. + + The user can add their own formats for saving and loading by passing their own objects + to the register_* functions. + + Args: + include_default_formats: If False, the built-in file formats will not be available. + Then only user-registered formats can be used. + path_manager: Used to customize how paths given as strings are interpreted. + """ + + def __init__( + self, + include_default_formats: bool = True, + path_manager: Optional[PathManager] = None, + ) -> None: + if path_manager is None: + self.path_manager = PathManager() + else: + self.path_manager = path_manager + + self.mesh_interpreters: Deque[MeshFormatInterpreter] = deque() + self.pointcloud_interpreters: Deque[PointcloudFormatInterpreter] = deque() + + if include_default_formats: + self.register_default_formats() + + def register_default_formats(self) -> None: + self.register_meshes_format(MeshObjFormat()) + self.register_meshes_format(MeshOffFormat()) + self.register_meshes_format(MeshPlyFormat()) + self.register_pointcloud_format(PointcloudPlyFormat()) + + def register_meshes_format(self, interpreter: MeshFormatInterpreter) -> None: + """ + Register a new interpreter for a new mesh file format. + + Args: + interpreter: the new interpreter to use, which must be an instance + of a class which inherits MeshFormatInterpreter. + """ + if not isinstance(interpreter, MeshFormatInterpreter): + raise ValueError("Invalid interpreter") + self.mesh_interpreters.appendleft(interpreter) + + def register_pointcloud_format( + self, interpreter: PointcloudFormatInterpreter + ) -> None: + """ + Register a new interpreter for a new point cloud file format. + + Args: + interpreter: the new interpreter to use, which must be an instance + of a class which inherits PointcloudFormatInterpreter. + """ + if not isinstance(interpreter, PointcloudFormatInterpreter): + raise ValueError("Invalid interpreter") + self.pointcloud_interpreters.appendleft(interpreter) + + def load_mesh( + self, + path: Union[str, Path], + include_textures: bool = True, + device: Device = "cpu", + **kwargs, + ) -> Meshes: + """ + Attempt to load a mesh from the given file, using a registered format. + Materials are not returned. If you have a .obj file with materials + you might want to load them with the load_obj function instead. + + Args: + path: file to read + include_textures: whether to try to load texture information + device: device on which to leave the data. + + Returns: + new Meshes object containing one mesh. + """ + for mesh_interpreter in self.mesh_interpreters: + mesh = mesh_interpreter.read( + path, + include_textures=include_textures, + path_manager=self.path_manager, + device=device, + **kwargs, + ) + if mesh is not None: + return mesh + + raise ValueError(f"No mesh interpreter found to read {path}.") + + def save_mesh( + self, + data: Meshes, + path: Union[str, Path], + binary: Optional[bool] = None, + include_textures: bool = True, + **kwargs, + ) -> None: + """ + Attempt to save a mesh to the given file, using a registered format. + + Args: + data: a 1-element Meshes + path: file to write + binary: If there is a choice, whether to save in a binary format. + include_textures: If textures are present, whether to try to save + them. + """ + if not isinstance(data, Meshes): + raise ValueError("Meshes object expected.") + + if len(data) != 1: + raise ValueError("Can only save a single mesh.") + + for mesh_interpreter in self.mesh_interpreters: + success = mesh_interpreter.save( + data, path, path_manager=self.path_manager, binary=binary, **kwargs + ) + if success: + return + + raise ValueError(f"No mesh interpreter found to write to {path}.") + + def load_pointcloud( + self, path: Union[str, Path], device: Device = "cpu", **kwargs + ) -> Pointclouds: + """ + Attempt to load a point cloud from the given file, using a registered format. + + Args: + path: file to read + device: Device (as str or torch.device) on which to load the data. + + Returns: + new Pointclouds object containing one mesh. + """ + for pointcloud_interpreter in self.pointcloud_interpreters: + pointcloud = pointcloud_interpreter.read( + path, path_manager=self.path_manager, device=device, **kwargs + ) + if pointcloud is not None: + return pointcloud + + raise ValueError(f"No point cloud interpreter found to read {path}.") + + def save_pointcloud( + self, + data: Pointclouds, + path: Union[str, Path], + binary: Optional[bool] = None, + **kwargs, + ) -> None: + """ + Attempt to save a point cloud to the given file, using a registered format. + + Args: + data: a 1-element Pointclouds + path: file to write + binary: If there is a choice, whether to save in a binary format. + """ + if not isinstance(data, Pointclouds): + raise ValueError("Pointclouds object expected.") + + if len(data) != 1: + raise ValueError("Can only save a single point cloud.") + + for pointcloud_interpreter in self.pointcloud_interpreters: + success = pointcloud_interpreter.save( + data, path, path_manager=self.path_manager, binary=binary, **kwargs + ) + if success: + return + + raise ValueError(f"No point cloud interpreter found to write to {path}.") diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/pluggable_formats.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/pluggable_formats.py new file mode 100644 index 0000000000000000000000000000000000000000..328a124dc12ec2fbf7e0e39e589d111414f32a2a --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/pluggable_formats.py @@ -0,0 +1,142 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +import pathlib +from typing import Optional, Tuple + +from iopath.common.file_io import PathManager +from pytorch3d.common.datatypes import Device +from pytorch3d.io.utils import PathOrStr +from pytorch3d.structures import Meshes, Pointclouds + + +""" +This module has the base classes which must be extended to define +an interpreter for loading and saving data in a particular format. +These can be registered on an IO object so that they can be used in +its load_* and save_* functions. +""" + + +def endswith(path: PathOrStr, suffixes: Tuple[str, ...]) -> bool: + """ + Returns whether the path ends with one of the given suffixes. + If `path` is not actually a path, returns True. This is useful + for allowing interpreters to bypass inappropriate paths, but + always accepting streams. + """ + if isinstance(path, pathlib.Path): + return path.suffix.lower() in suffixes + if isinstance(path, str): + return path.lower().endswith(suffixes) + return True + + +class MeshFormatInterpreter: + """ + This is a base class for an interpreter which can read or write + a mesh in a particular format. + """ + + def read( + self, + path: PathOrStr, + include_textures: bool, + device: Device, + path_manager: PathManager, + **kwargs, + ) -> Optional[Meshes]: + """ + Read the data from the specified file and return it as + a Meshes object. + + Args: + path: path to load. + include_textures: whether to try to load texture information. + device: torch.device to load data on to. + path_manager: PathManager to interpret the path. + + Returns: + None if self is not the appropriate object to interpret the given + path. + Otherwise, the read Meshes object. + """ + raise NotImplementedError() + + def save( + self, + data: Meshes, + path: PathOrStr, + path_manager: PathManager, + binary: Optional[bool], + **kwargs, + ) -> bool: + """ + Save the given Meshes object to the given path. + + Args: + data: mesh to save + path: path to save to, which may be overwritten. + path_manager: PathManager to interpret the path. + binary: If there is a choice, whether to save in a binary format. + + Returns: + False: if self is not the appropriate object to write to the given path. + True: on success. + """ + raise NotImplementedError() + + +class PointcloudFormatInterpreter: + """ + This is a base class for an interpreter which can read or write + a point cloud in a particular format. + """ + + def read( + self, path: PathOrStr, device: Device, path_manager: PathManager, **kwargs + ) -> Optional[Pointclouds]: + """ + Read the data from the specified file and return it as + a Pointclouds object. + + Args: + path: path to load. + device: torch.device to load data on to. + path_manager: PathManager to interpret the path. + + Returns: + None if self is not the appropriate object to interpret the given + path. + Otherwise, the read Pointclouds object. + """ + raise NotImplementedError() + + def save( + self, + data: Pointclouds, + path: PathOrStr, + path_manager: PathManager, + binary: Optional[bool], + **kwargs, + ) -> bool: + """ + Save the given Pointclouds object to the given path. + + Args: + data: point cloud object to save + path: path to save to, which may be overwritten. + path_manager: PathManager to interpret the path. + binary: If there is a choice, whether to save in a binary format. + + Returns: + False: if self is not the appropriate object to write to the given path. + True: on success. + """ + raise NotImplementedError() diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/ply_io.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/ply_io.py new file mode 100644 index 0000000000000000000000000000000000000000..99207e6fd1903cf63c887f211f8f824cb3da3dbe --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/ply_io.py @@ -0,0 +1,1539 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +""" +This module implements utility functions for loading and saving +meshes and point clouds as PLY files. +""" +import itertools +import os +import struct +import sys +import warnings +from collections import namedtuple +from dataclasses import asdict, dataclass +from io import BytesIO, TextIOBase +from typing import List, Optional, Tuple + +import numpy as np +import torch +from iopath.common.file_io import PathManager +from pytorch3d.io.utils import ( + _check_faces_indices, + _make_tensor, + _open_file, + _read_image, + PathOrStr, +) +from pytorch3d.renderer import TexturesUV, TexturesVertex +from pytorch3d.structures import Meshes, Pointclouds + +from .pluggable_formats import ( + endswith, + MeshFormatInterpreter, + PointcloudFormatInterpreter, +) + + +_PlyTypeData = namedtuple("_PlyTypeData", "size struct_char np_type") + +_PLY_TYPES = { + "char": _PlyTypeData(1, "b", np.byte), + "uchar": _PlyTypeData(1, "B", np.ubyte), + "short": _PlyTypeData(2, "h", np.short), + "ushort": _PlyTypeData(2, "H", np.ushort), + "int": _PlyTypeData(4, "i", np.int32), + "uint": _PlyTypeData(4, "I", np.uint32), + "float": _PlyTypeData(4, "f", np.float32), + "double": _PlyTypeData(8, "d", np.float64), + "int8": _PlyTypeData(1, "b", np.byte), + "uint8": _PlyTypeData(1, "B", np.ubyte), + "int16": _PlyTypeData(2, "h", np.short), + "uint16": _PlyTypeData(2, "H", np.ushort), + "int32": _PlyTypeData(4, "i", np.int32), + "uint32": _PlyTypeData(4, "I", np.uint32), + "float32": _PlyTypeData(4, "f", np.float32), + "float64": _PlyTypeData(8, "d", np.float64), +} + +_Property = namedtuple("_Property", "name data_type list_size_type") + + +class _PlyElementType: + """ + Description of an element of a Ply file. + Members: + self.properties: (List[_Property]) description of all the properties. + Each one contains a name and data type. + self.count: (int) number of such elements in the file + self.name: (str) name of the element + """ + + def __init__(self, name: str, count: int) -> None: + self.name = name + self.count = count + self.properties: List[_Property] = [] + + def add_property( + self, name: str, data_type: str, list_size_type: Optional[str] = None + ): + """Adds a new property. + + Args: + name: (str) name of the property. + data_type: (str) PLY data type. + list_size_type: (str) PLY data type of the list size, or None if not + a list. + """ + for property in self.properties: + if property.name == name: + msg = "Cannot have two properties called %s in %s." + raise ValueError(msg % (name, self.name)) + self.properties.append(_Property(name, data_type, list_size_type)) + + def is_fixed_size(self) -> bool: + """Return whether the Element has no list properties + + Returns: + True if none of the properties are lists. + """ + for property in self.properties: + if property.list_size_type is not None: + return False + return True + + def is_constant_type_fixed_size(self) -> bool: + """Return whether the Element has all properties of the same non-list + type. + + Returns: + True if none of the properties are lists and all the properties + share a type. + """ + if not self.is_fixed_size(): + return False + first_type = _PLY_TYPES[self.properties[0].data_type] + for property in self.properties: + if _PLY_TYPES[property.data_type] != first_type: + return False + return True + + def try_constant_list(self) -> bool: + """Whether the element is just a single list, which might have a + constant size, and therefore we could try to parse quickly with numpy. + + Returns: + True if the only property is a list. + """ + if len(self.properties) != 1: + return False + if self.properties[0].list_size_type is None: + return False + return True + + +class _PlyHeader: + def __init__(self, f) -> None: + """ + Load a header of a Ply file from a file-like object. + Members: + self.elements: (List[_PlyElementType]) element description + self.ascii: (bool) Whether in ascii format + self.big_endian: (bool) (if not ascii) whether big endian + self.obj_info: (List[str]) arbitrary extra data + self.comments: (List[str]) comments + + Args: + f: file-like object. + """ + if f.readline() not in [b"ply\n", b"ply\r\n", "ply\n"]: + raise ValueError("Invalid file header.") + seen_format = False + self.elements: List[_PlyElementType] = [] + self.comments: List[str] = [] + self.obj_info: List[str] = [] + while True: + line = f.readline() + if isinstance(line, bytes): + line = line.decode("ascii") + line = line.strip() + if line == "end_header": + if not self.elements: + raise ValueError("No elements found.") + if not self.elements[-1].properties: + raise ValueError("Found an element with no properties.") + if not seen_format: + raise ValueError("No format line found.") + break + if not seen_format: + if line == "format ascii 1.0": + seen_format = True + self.ascii = True + continue + if line == "format binary_little_endian 1.0": + seen_format = True + self.ascii = False + self.big_endian = False + continue + if line == "format binary_big_endian 1.0": + seen_format = True + self.ascii = False + self.big_endian = True + continue + if line.startswith("format"): + raise ValueError("Invalid format line.") + if line.startswith("comment "): + self.comments.append(line[8:]) + continue + if line.startswith("comment") or len(line) == 0: + continue + if line.startswith("element"): + self._parse_element(line) + continue + if line.startswith("obj_info "): + self.obj_info.append(line[9:]) + continue + if line.startswith("property"): + self._parse_property(line) + continue + raise ValueError("Invalid line: %s." % line) + + def _parse_property(self, line: str): + """ + Decode a ply file header property line. + + Args: + line: (str) the ply file's line. + """ + if not self.elements: + raise ValueError("Encountered property before any element.") + items = line.split(" ") + if len(items) not in [3, 5]: + raise ValueError("Invalid line: %s" % line) + datatype = items[1] + name = items[-1] + if datatype == "list": + datatype = items[3] + list_size_type = items[2] + if list_size_type not in _PLY_TYPES: + raise ValueError("Invalid datatype: %s" % list_size_type) + else: + list_size_type = None + if datatype not in _PLY_TYPES: + raise ValueError("Invalid datatype: %s" % datatype) + self.elements[-1].add_property(name, datatype, list_size_type) + + def _parse_element(self, line: str): + """ + Decode a ply file header element line. + + Args: + line: (str) the ply file's line. + """ + if self.elements and not self.elements[-1].properties: + raise ValueError("Found an element with no properties.") + items = line.split(" ") + if len(items) != 3: + raise ValueError("Invalid line: %s" % line) + try: + count = int(items[2]) + except ValueError: + msg = "Number of items for %s was not a number." + raise ValueError(msg % items[1]) from None + self.elements.append(_PlyElementType(items[1], count)) + + +def _read_ply_fixed_size_element_ascii(f, definition: _PlyElementType): + """ + Given an element which has no lists and one type, read the + corresponding data. + + For example + + element vertex 8 + property float x + property float y + property float z + + Args: + f: file-like object being read. + definition: The element object which describes what we are reading. + + Returns: + 1-element list containing a 2D numpy array corresponding to the data. + The rows are the different values. There is one column for each property. + """ + np_type = _PLY_TYPES[definition.properties[0].data_type].np_type + old_offset = f.tell() + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", message=".* Empty input file.*", category=UserWarning + ) + data = np.loadtxt( + f, dtype=np_type, comments=None, ndmin=2, max_rows=definition.count + ) + if not len(data): # np.loadtxt() seeks even on empty data + f.seek(old_offset) + if data.shape[1] != len(definition.properties): + raise ValueError("Inconsistent data for %s." % definition.name) + if data.shape[0] != definition.count: + raise ValueError("Not enough data for %s." % definition.name) + return [data] + + +def _read_ply_nolist_element_ascii(f, definition: _PlyElementType): + """ + Given an element which has no lists and multiple types, read the + corresponding data, by loading all the data as float64 and converting + the relevant parts later. + + For example, given + + element vertex 8 + property float x + property float y + property float z + property uchar red + property uchar green + property uchar blue + + the output will have two arrays, the first containing (x,y,z) + and the second (red,green,blue). + + Args: + f: file-like object being read. + definition: The element object which describes what we are reading. + + Returns: + List of 2D numpy arrays corresponding to the data. + """ + old_offset = f.tell() + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", message=".* Empty input file.*", category=UserWarning + ) + data = np.loadtxt( + f, dtype=np.float64, comments=None, ndmin=2, max_rows=definition.count + ) + if not len(data): # np.loadtxt() seeks even on empty data + f.seek(old_offset) + if data.shape[1] != len(definition.properties): + raise ValueError("Inconsistent data for %s." % definition.name) + if data.shape[0] != definition.count: + raise ValueError("Not enough data for %s." % definition.name) + pieces = [] + offset = 0 + for dtype, it in itertools.groupby(p.data_type for p in definition.properties): + count = sum(1 for _ in it) + end_offset = offset + count + piece = data[:, offset:end_offset].astype(_PLY_TYPES[dtype].np_type) + pieces.append(piece) + offset = end_offset + return pieces + + +def _try_read_ply_constant_list_ascii(f, definition: _PlyElementType): + """ + If definition is an element which is a single list, attempt to read the + corresponding data assuming every value has the same length. + If the data is ragged, return None and leave f undisturbed. + + For example, if the element is + + element face 2 + property list uchar int vertex_index + + and the data is + + 4 0 1 2 3 + 4 7 6 5 4 + + then the function will return + + [[0, 1, 2, 3], + [7, 6, 5, 4]] + + but if the data is + + 4 0 1 2 3 + 3 6 5 4 + + then the function will return None. + + Args: + f: file-like object being read. + definition: The element object which describes what we are reading. + + Returns: + If every element has the same size, 2D numpy array corresponding to the + data. The rows are the different values. Otherwise None. + """ + np_type = _PLY_TYPES[definition.properties[0].data_type].np_type + old_offset = f.tell() + try: + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", message=".* Empty input file.*", category=UserWarning + ) + data = np.loadtxt( + f, dtype=np_type, comments=None, ndmin=2, max_rows=definition.count + ) + except ValueError: + f.seek(old_offset) + return None + if not len(data): # np.loadtxt() seeks even on empty data + f.seek(old_offset) + if (data[:, 0] != data.shape[1] - 1).any(): + msg = "A line of %s data did not have the specified length." + raise ValueError(msg % definition.name) + if data.shape[0] != definition.count: + raise ValueError("Not enough data for %s." % definition.name) + return data[:, 1:] + + +def _parse_heterogeneous_property_ascii(datum, line_iter, property: _Property): + """ + Read a general data property from an ascii .ply file. + + Args: + datum: list to append the single value to. That value will be a numpy + array if the property is a list property, otherwise an int or + float. + line_iter: iterator to words on the line from which we read. + property: the property object describing the property we are reading. + """ + value = next(line_iter, None) + if value is None: + raise ValueError("Too little data for an element.") + if property.list_size_type is None: + try: + if property.data_type in ["double", "float"]: + datum.append(float(value)) + else: + datum.append(int(value)) + except ValueError: + raise ValueError("Bad numerical data.") from None + else: + try: + length = int(value) + except ValueError: + raise ValueError("A list length was not a number.") from None + list_value = np.zeros(length, dtype=_PLY_TYPES[property.data_type].np_type) + for i in range(length): + inner_value = next(line_iter, None) + if inner_value is None: + raise ValueError("Too little data for an element.") + try: + list_value[i] = float(inner_value) + except ValueError: + raise ValueError("Bad numerical data.") from None + datum.append(list_value) + + +def _read_ply_element_ascii(f, definition: _PlyElementType): + """ + Decode all instances of a single element from an ascii .ply file. + + Args: + f: file-like object being read. + definition: The element object which describes what we are reading. + + Returns: + In simple cases where every element has the same size, 2D numpy array + corresponding to the data. The rows are the different values. + Otherwise a list of lists of values, where the outer list is + each occurrence of the element, and the inner lists have one value per + property. + """ + if not definition.count: + return [] + if definition.is_constant_type_fixed_size(): + return _read_ply_fixed_size_element_ascii(f, definition) + if definition.is_fixed_size(): + return _read_ply_nolist_element_ascii(f, definition) + if definition.try_constant_list(): + data = _try_read_ply_constant_list_ascii(f, definition) + if data is not None: + return data + + # We failed to read the element as a lump, must process each line manually. + data = [] + for _i in range(definition.count): + line_string = f.readline() + if line_string == "": + raise ValueError("Not enough data for %s." % definition.name) + datum = [] + line_iter = iter(line_string.strip().split()) + for property in definition.properties: + _parse_heterogeneous_property_ascii(datum, line_iter, property) + data.append(datum) + if next(line_iter, None) is not None: + raise ValueError("Too much data for an element.") + return data + + +def _read_raw_array( + f, aim: str, length: int, dtype: type = np.uint8, dtype_size: int = 1 +): + """ + Read [length] elements from a file. + + Args: + f: file object + aim: name of target for error message + length: number of elements + dtype: numpy type + dtype_size: number of bytes per element. + + Returns: + new numpy array + """ + + if isinstance(f, BytesIO): + # np.fromfile is faster but won't work on a BytesIO + needed_bytes = length * dtype_size + bytes_data = bytearray(needed_bytes) + n_bytes_read = f.readinto(bytes_data) + if n_bytes_read != needed_bytes: + raise ValueError("Not enough data for %s." % aim) + data = np.frombuffer(bytes_data, dtype=dtype) + else: + data = np.fromfile(f, dtype=dtype, count=length) + if data.shape[0] != length: + raise ValueError("Not enough data for %s." % aim) + return data + + +def _read_ply_fixed_size_element_binary( + f, definition: _PlyElementType, big_endian: bool +): + """ + Given an element which has no lists and one type, read the + corresponding data. + + For example + + element vertex 8 + property float x + property float y + property float z + + + Args: + f: file-like object being read. + definition: The element object which describes what we are reading. + big_endian: (bool) whether the document is encoded as big endian. + + Returns: + 1-element list containing a 2D numpy array corresponding to the data. + The rows are the different values. There is one column for each property. + """ + ply_type = _PLY_TYPES[definition.properties[0].data_type] + np_type = ply_type.np_type + type_size = ply_type.size + needed_length = definition.count * len(definition.properties) + data = _read_raw_array(f, definition.name, needed_length, np_type, type_size) + + if (sys.byteorder == "big") != big_endian: + data = data.byteswap() + return [data.reshape(definition.count, len(definition.properties))] + + +def _read_ply_element_binary_nolists(f, definition: _PlyElementType, big_endian: bool): + """ + Given an element which has no lists, read the corresponding data as tuple + of numpy arrays, one for each set of adjacent columns with the same type. + + For example, given + + element vertex 8 + property float x + property float y + property float z + property uchar red + property uchar green + property uchar blue + + the output will have two arrays, the first containing (x,y,z) + and the second (red,green,blue). + + Args: + f: file-like object being read. + definition: The element object which describes what we are reading. + big_endian: (bool) whether the document is encoded as big endian. + + Returns: + List of 2D numpy arrays corresponding to the data. The rows are the different + values. + """ + size = sum(_PLY_TYPES[prop.data_type].size for prop in definition.properties) + needed_bytes = size * definition.count + data = _read_raw_array(f, definition.name, needed_bytes).reshape(-1, size) + offset = 0 + pieces = [] + for dtype, it in itertools.groupby(p.data_type for p in definition.properties): + count = sum(1 for _ in it) + bytes_each = count * _PLY_TYPES[dtype].size + end_offset = offset + bytes_each + + # what we want to do is + # piece = data[:, offset:end_offset].view(_PLY_TYPES[dtype].np_type) + # but it fails in the general case + # because of https://github.com/numpy/numpy/issues/9496. + piece = np.lib.stride_tricks.as_strided( + data[:1, offset:end_offset].view(_PLY_TYPES[dtype].np_type), + shape=(definition.count, count), + strides=(data.strides[0], _PLY_TYPES[dtype].size), + ) + + if (sys.byteorder == "big") != big_endian: + piece = piece.byteswap() + pieces.append(piece) + offset = end_offset + return pieces + + +def _try_read_ply_constant_list_binary( + f, definition: _PlyElementType, big_endian: bool +): + """ + If definition is an element which is a single list, attempt to read the + corresponding data assuming every value has the same length. + If the data is ragged, return None and leave f undisturbed. + + For example, if the element is + + element face 2 + property list uchar int vertex_index + + and the data is + + 4 0 1 2 3 + 4 7 6 5 4 + + then the function will return + + [[0, 1, 2, 3], + [7, 6, 5, 4]] + + but if the data is + + 4 0 1 2 3 + 3 6 5 4 + + then the function will return None. + + Args: + f: file-like object being read. + definition: The element object which describes what we are reading. + big_endian: (bool) whether the document is encoded as big endian. + + Returns: + If every element has the same size, 2D numpy array corresponding to the + data. The rows are the different values. Otherwise None. + """ + property = definition.properties[0] + endian_str = ">" if big_endian else "<" + length_format = endian_str + _PLY_TYPES[property.list_size_type].struct_char + length_struct = struct.Struct(length_format) + + def get_length(): + bytes_data = f.read(length_struct.size) + if len(bytes_data) != length_struct.size: + raise ValueError("Not enough data for %s." % definition.name) + [length] = length_struct.unpack(bytes_data) + return length + + old_offset = f.tell() + + length = get_length() + np_type = _PLY_TYPES[definition.properties[0].data_type].np_type + type_size = _PLY_TYPES[definition.properties[0].data_type].size + data_size = type_size * length + + output = np.zeros((definition.count, length), dtype=np_type) + + for i in range(definition.count): + bytes_data = f.read(data_size) + if len(bytes_data) != data_size: + raise ValueError("Not enough data for %s" % definition.name) + output[i] = np.frombuffer(bytes_data, dtype=np_type) + if i + 1 == definition.count: + break + if length != get_length(): + f.seek(old_offset) + return None + if (sys.byteorder == "big") != big_endian: + output = output.byteswap() + + return output + + +def _read_ply_element_binary(f, definition: _PlyElementType, big_endian: bool) -> list: + """ + Decode all instances of a single element from a binary .ply file. + + Args: + f: file-like object being read. + definition: The element object which describes what we are reading. + big_endian: (bool) whether the document is encoded as big endian. + + Returns: + In simple cases where every element has the same size, 2D numpy array + corresponding to the data. The rows are the different values. + Otherwise a list of lists/tuples of values, where the outer list is + each occurrence of the element, and the inner lists have one value per + property. + """ + if not definition.count: + return [] + + if definition.is_constant_type_fixed_size(): + return _read_ply_fixed_size_element_binary(f, definition, big_endian) + if definition.is_fixed_size(): + return _read_ply_element_binary_nolists(f, definition, big_endian) + if definition.try_constant_list(): + data = _try_read_ply_constant_list_binary(f, definition, big_endian) + if data is not None: + return data + + # We failed to read the element as a lump, must process each line manually. + endian_str = ">" if big_endian else "<" + property_structs = [] + for property in definition.properties: + initial_type = property.list_size_type or property.data_type + property_structs.append( + struct.Struct(endian_str + _PLY_TYPES[initial_type].struct_char) + ) + + data = [] + for _i in range(definition.count): + datum = [] + for property, property_struct in zip(definition.properties, property_structs): + size = property_struct.size + initial_data = f.read(size) + if len(initial_data) != size: + raise ValueError("Not enough data for %s" % definition.name) + [initial] = property_struct.unpack(initial_data) + if property.list_size_type is None: + datum.append(initial) + else: + type_size = _PLY_TYPES[property.data_type].size + needed_bytes = type_size * initial + list_data = f.read(needed_bytes) + if len(list_data) != needed_bytes: + raise ValueError("Not enough data for %s" % definition.name) + np_type = _PLY_TYPES[property.data_type].np_type + list_np = np.frombuffer(list_data, dtype=np_type) + if (sys.byteorder == "big") != big_endian: + list_np = list_np.byteswap() + datum.append(list_np) + data.append(datum) + return data + + +def _load_ply_raw_stream(f) -> Tuple[_PlyHeader, dict]: + """ + Implementation for _load_ply_raw which takes a stream. + + Args: + f: A binary or text file-like object. + + Returns: + header: A _PlyHeader object describing the metadata in the ply file. + elements: A dictionary of element names to values. If an element is regular, in + the sense of having no lists or being one uniformly-sized list, then the + value will be a 2D numpy array. If not, it is a list of the relevant + property values. + """ + + header = _PlyHeader(f) + elements = {} + if header.ascii: + for element in header.elements: + elements[element.name] = _read_ply_element_ascii(f, element) + else: + if isinstance(f, TextIOBase): + raise ValueError( + "Cannot safely read a binary ply file using a Text stream." + ) + big = header.big_endian + for element in header.elements: + elements[element.name] = _read_ply_element_binary(f, element, big) + end = f.read().strip() + if len(end) != 0: + raise ValueError("Extra data at end of file: " + str(end[:20])) + return header, elements + + +def _load_ply_raw(f, path_manager: PathManager) -> Tuple[_PlyHeader, dict]: + """ + Load the data from a .ply file. + + Args: + f: A binary or text file-like object (with methods read, readline, + tell and seek), a pathlib path or a string containing a file name. + If the ply file is binary, a text stream is not supported. + It is recommended to use a binary stream. + path_manager: PathManager for loading if f is a str. + + Returns: + header: A _PlyHeader object describing the metadata in the ply file. + elements: A dictionary of element names to values. If an element is + regular, in the sense of having no lists or being one + uniformly-sized list, then the value will be a 2D numpy array. + If it has no lists but more than one type, it will be a list of arrays. + If not, it is a list of the relevant property values. + """ + with _open_file(f, path_manager, "rb") as f: + header, elements = _load_ply_raw_stream(f) + return header, elements + + +@dataclass(frozen=True) +class _VertsColumnIndices: + """ + Contains the relevant layout of the verts section of file being read. + Members + point_idxs: List[int] of 3 point columns. + color_idxs: List[int] of 3 color columns if they are present, + otherwise None. + color_scale: value to scale colors by. + normal_idxs: List[int] of 3 normals columns if they are present, + otherwise None. + """ + + point_idxs: List[int] + color_idxs: Optional[List[int]] + color_scale: float + normal_idxs: Optional[List[int]] + texture_uv_idxs: Optional[List[int]] + + +def _get_verts_column_indices( + vertex_head: _PlyElementType, +) -> _VertsColumnIndices: + """ + Get the columns of verts, verts_colors, and verts_normals in the vertex + element of a parsed ply file, together with a color scale factor. + When the colors are in byte format, they are scaled from 0..255 to [0,1]. + Otherwise they are not scaled. + + For example, if the vertex element looks as follows: + + element vertex 892 + property double x + property double y + property double z + property double nx + property double ny + property double nz + property uchar red + property uchar green + property uchar blue + property double texture_u + property double texture_v + + then the return value will be ([0,1,2], [6,7,8], 1.0/255, [3,4,5]) + + Args: + vertex_head: as returned from load_ply_raw. + + Returns: + _VertsColumnIndices object + """ + point_idxs: List[Optional[int]] = [None, None, None] + color_idxs: List[Optional[int]] = [None, None, None] + normal_idxs: List[Optional[int]] = [None, None, None] + texture_uv_idxs: List[Optional[int]] = [None, None] + for i, prop in enumerate(vertex_head.properties): + if prop.list_size_type is not None: + raise ValueError("Invalid vertices in file: did not expect list.") + for j, letter in enumerate(["x", "y", "z"]): + if prop.name == letter: + point_idxs[j] = i + for j, name in enumerate(["red", "green", "blue"]): + if prop.name == name: + color_idxs[j] = i + for j, name in enumerate(["nx", "ny", "nz"]): + if prop.name == name: + normal_idxs[j] = i + for j, name in enumerate(["texture_u", "texture_v"]): + if prop.name == name: + texture_uv_idxs[j] = i + if None in point_idxs: + raise ValueError("Invalid vertices in file.") + color_scale = 1.0 + if all( + idx is not None and _PLY_TYPES[vertex_head.properties[idx].data_type].size == 1 + for idx in color_idxs + ): + color_scale = 1.0 / 255 + return _VertsColumnIndices( + point_idxs=point_idxs, + color_idxs=None if None in color_idxs else color_idxs, + color_scale=color_scale, + normal_idxs=None if None in normal_idxs else normal_idxs, + texture_uv_idxs=None if None in texture_uv_idxs else texture_uv_idxs, + ) + + +@dataclass(frozen=True) +class _VertsData: + """ + Contains the data of the verts section of file being read. + Members: + verts: FloatTensor of shape (V, 3). + verts_colors: None or FloatTensor of shape (V, 3). + verts_normals: None or FloatTensor of shape (V, 3). + """ + + verts: torch.Tensor + verts_colors: Optional[torch.Tensor] = None + verts_normals: Optional[torch.Tensor] = None + verts_texture_uvs: Optional[torch.Tensor] = None + + +def _get_verts(header: _PlyHeader, elements: dict) -> _VertsData: + """ + Get the vertex locations, colors and normals from a parsed ply file. + + Args: + header, elements: as returned from load_ply_raw. + + Returns: + _VertsData object + """ + + vertex = elements.get("vertex", None) + if vertex is None: + raise ValueError("The ply file has no vertex element.") + if not isinstance(vertex, list): + raise ValueError("Invalid vertices in file.") + vertex_head = next(head for head in header.elements if head.name == "vertex") + + column_idxs = _get_verts_column_indices(vertex_head) + + # Case of no vertices + if vertex_head.count == 0: + verts = torch.zeros((0, 3), dtype=torch.float32) + if column_idxs.color_idxs is None: + return _VertsData(verts=verts) + return _VertsData( + verts=verts, verts_colors=torch.zeros((0, 3), dtype=torch.float32) + ) + + # Simple case where the only data is the vertices themselves + if ( + len(vertex) == 1 + and isinstance(vertex[0], np.ndarray) + and vertex[0].ndim == 2 + and vertex[0].shape[1] == 3 + ): + return _VertsData(verts=_make_tensor(vertex[0], cols=3, dtype=torch.float32)) + + vertex_colors = None + vertex_normals = None + vertex_texture_uvs = None + + if len(vertex) == 1: + # This is the case where the whole vertex element has one type, + # so it was read as a single array and we can index straight into it. + verts = torch.tensor(vertex[0][:, column_idxs.point_idxs], dtype=torch.float32) + if column_idxs.color_idxs is not None: + vertex_colors = column_idxs.color_scale * torch.tensor( + vertex[0][:, column_idxs.color_idxs], dtype=torch.float32 + ) + if column_idxs.normal_idxs is not None: + vertex_normals = torch.tensor( + vertex[0][:, column_idxs.normal_idxs], dtype=torch.float32 + ) + if column_idxs.texture_uv_idxs is not None: + vertex_texture_uvs = torch.tensor( + vertex[0][:, column_idxs.texture_uv_idxs], dtype=torch.float32 + ) + else: + # The vertex element is heterogeneous. It was read as several arrays, + # part by part, where a part is a set of properties with the same type. + # For each property (=column in the file), we store in + # prop_to_partnum_col its partnum (i.e. the index of what part it is + # in) and its column number (its index within its part). + prop_to_partnum_col = [ + (partnum, col) + for partnum, array in enumerate(vertex) + for col in range(array.shape[1]) + ] + verts = torch.empty(size=(vertex_head.count, 3), dtype=torch.float32) + for axis in range(3): + partnum, col = prop_to_partnum_col[column_idxs.point_idxs[axis]] + verts.numpy()[:, axis] = vertex[partnum][:, col] + # Note that in the previous line, we made the assignment + # as numpy arrays by casting verts. If we took the (more + # obvious) method of converting the right hand side to + # torch, then we might have an extra data copy because + # torch wants contiguity. The code would be like: + # if not vertex[partnum].flags["C_CONTIGUOUS"]: + # vertex[partnum] = np.ascontiguousarray(vertex[partnum]) + # verts[:, axis] = torch.tensor((vertex[partnum][:, col])) + if column_idxs.color_idxs is not None: + vertex_colors = torch.empty( + size=(vertex_head.count, 3), dtype=torch.float32 + ) + for color in range(3): + partnum, col = prop_to_partnum_col[column_idxs.color_idxs[color]] + vertex_colors.numpy()[:, color] = vertex[partnum][:, col] + vertex_colors *= column_idxs.color_scale + if column_idxs.normal_idxs is not None: + vertex_normals = torch.empty( + size=(vertex_head.count, 3), dtype=torch.float32 + ) + for axis in range(3): + partnum, col = prop_to_partnum_col[column_idxs.normal_idxs[axis]] + vertex_normals.numpy()[:, axis] = vertex[partnum][:, col] + if column_idxs.texture_uv_idxs is not None: + vertex_texture_uvs = torch.empty( + size=(vertex_head.count, 2), + dtype=torch.float32, + ) + for axis in range(2): + partnum, col = prop_to_partnum_col[column_idxs.texture_uv_idxs[axis]] + vertex_texture_uvs.numpy()[:, axis] = vertex[partnum][:, col] + return _VertsData( + verts=verts, + verts_colors=vertex_colors, + verts_normals=vertex_normals, + verts_texture_uvs=vertex_texture_uvs, + ) + + +@dataclass(frozen=True) +class _PlyData: + """ + Contains the data from a PLY file which has been read. + Members: + header: _PlyHeader of file metadata from the header + verts: FloatTensor of shape (V, 3). + faces: None or LongTensor of vertex indices, shape (F, 3). + verts_colors: None or FloatTensor of shape (V, 3). + verts_normals: None or FloatTensor of shape (V, 3). + """ + + header: _PlyHeader + verts: torch.Tensor + faces: Optional[torch.Tensor] + verts_colors: Optional[torch.Tensor] + verts_normals: Optional[torch.Tensor] + verts_texture_uvs: Optional[torch.Tensor] + + +def _load_ply(f, *, path_manager: PathManager) -> _PlyData: + """ + Load the data from a .ply file. + + Args: + f: A binary or text file-like object (with methods read, readline, + tell and seek), a pathlib path or a string containing a file name. + If the ply file is in the binary ply format rather than the text + ply format, then a text stream is not supported. + It is easiest to use a binary stream in all cases. + path_manager: PathManager for loading if f is a str. + + Returns: + _PlyData object + """ + header, elements = _load_ply_raw(f, path_manager=path_manager) + + verts_data = _get_verts(header, elements) + + face = elements.get("face", None) + if face is not None: + face_head = next(head for head in header.elements if head.name == "face") + if ( + len(face_head.properties) != 1 + or face_head.properties[0].list_size_type is None + ): + raise ValueError("Unexpected form of faces data.") + # face_head.properties[0].name is usually "vertex_index" or "vertex_indices" + # but we don't need to enforce this. + + if face is None: + faces = None + elif not len(face): + # pyre is happier when this condition is not joined to the + # previous one with `or`. + faces = None + elif isinstance(face, np.ndarray) and face.ndim == 2: # Homogeneous elements + if face.shape[1] < 3: + raise ValueError("Faces must have at least 3 vertices.") + face_arrays = [face[:, [0, i + 1, i + 2]] for i in range(face.shape[1] - 2)] + faces = torch.LongTensor(np.vstack(face_arrays).astype(np.int64)) + else: + face_list = [] + for (face_item,) in face: + if face_item.ndim != 1: + raise ValueError("Bad face data.") + if face_item.shape[0] < 3: + raise ValueError("Faces must have at least 3 vertices.") + for i in range(face_item.shape[0] - 2): + face_list.append([face_item[0], face_item[i + 1], face_item[i + 2]]) + faces = torch.tensor(face_list, dtype=torch.int64) + + if faces is not None: + _check_faces_indices(faces, max_index=verts_data.verts.shape[0]) + + return _PlyData(**asdict(verts_data), faces=faces, header=header) + + +def load_ply( + f, *, path_manager: Optional[PathManager] = None +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Load the verts and faces from a .ply file. + Note that the preferred way to load data from such a file + is to use the IO.load_mesh and IO.load_pointcloud functions, + which can read more of the data. + + Example .ply file format:: + + ply + format ascii 1.0 { ascii/binary, format version number } + comment made by Greg Turk { comments keyword specified, like all lines } + comment this file is a cube + element vertex 8 { define "vertex" element, 8 of them in file } + property float x { vertex contains float "x" coordinate } + property float y { y coordinate is also a vertex property } + property float z { z coordinate, too } + element face 6 { there are 6 "face" elements in the file } + property list uchar int vertex_index { "vertex_indices" is a list of ints } + end_header { delimits the end of the header } + 0 0 0 { start of vertex list } + 0 0 1 + 0 1 1 + 0 1 0 + 1 0 0 + 1 0 1 + 1 1 1 + 1 1 0 + 4 0 1 2 3 { start of face list } + 4 7 6 5 4 + 4 0 4 5 1 + 4 1 5 6 2 + 4 2 6 7 3 + 4 3 7 4 0 + + Args: + f: A binary or text file-like object (with methods read, readline, + tell and seek), a pathlib path or a string containing a file name. + If the ply file is in the binary ply format rather than the text + ply format, then a text stream is not supported. + It is easiest to use a binary stream in all cases. + path_manager: PathManager for loading if f is a str. + + Returns: + verts: FloatTensor of shape (V, 3). + faces: LongTensor of vertex indices, shape (F, 3). + """ + + if path_manager is None: + path_manager = PathManager() + data = _load_ply(f, path_manager=path_manager) + faces = data.faces + if faces is None: + faces = torch.zeros(0, 3, dtype=torch.int64) + + return data.verts, faces + + +def _write_ply_header( + f, + *, + verts: torch.Tensor, + faces: Optional[torch.LongTensor], + verts_normals: Optional[torch.Tensor], + verts_colors: Optional[torch.Tensor], + ascii: bool, + colors_as_uint8: bool, +) -> None: + """ + Internal implementation for writing header when saving to a .ply file. + + Args: + f: File object to which the 3D data should be written. + verts: FloatTensor of shape (V, 3) giving vertex coordinates. + faces: LongTensor of shape (F, 3) giving faces. + verts_normals: FloatTensor of shape (V, 3) giving vertex normals. + verts_colors: FloatTensor of shape (V, 3) giving vertex colors. + ascii: (bool) whether to use the ascii ply format. + colors_as_uint8: Whether to save colors as numbers in the range + [0, 255] instead of float32. + """ + assert not len(verts) or (verts.dim() == 2 and verts.size(1) == 3) + assert faces is None or not len(faces) or (faces.dim() == 2 and faces.size(1) == 3) + assert verts_normals is None or ( + verts_normals.dim() == 2 and verts_normals.size(1) == 3 + ) + assert verts_colors is None or ( + verts_colors.dim() == 2 and verts_colors.size(1) == 3 + ) + + if ascii: + f.write(b"ply\nformat ascii 1.0\n") + elif sys.byteorder == "big": + f.write(b"ply\nformat binary_big_endian 1.0\n") + else: + f.write(b"ply\nformat binary_little_endian 1.0\n") + f.write(f"element vertex {verts.shape[0]}\n".encode("ascii")) + f.write(b"property float x\n") + f.write(b"property float y\n") + f.write(b"property float z\n") + if verts_normals is not None: + f.write(b"property float nx\n") + f.write(b"property float ny\n") + f.write(b"property float nz\n") + if verts_colors is not None: + color_ply_type = b"uchar" if colors_as_uint8 else b"float" + for color in (b"red", b"green", b"blue"): + f.write(b"property " + color_ply_type + b" " + color + b"\n") + if len(verts) and faces is not None: + f.write(f"element face {faces.shape[0]}\n".encode("ascii")) + f.write(b"property list uchar int vertex_index\n") + f.write(b"end_header\n") + + +def _save_ply( + f, + *, + verts: torch.Tensor, + faces: Optional[torch.LongTensor], + verts_normals: Optional[torch.Tensor], + verts_colors: Optional[torch.Tensor], + ascii: bool, + decimal_places: Optional[int] = None, + colors_as_uint8: bool, +) -> None: + """ + Internal implementation for saving 3D data to a .ply file. + + Args: + f: File object to which the 3D data should be written. + verts: FloatTensor of shape (V, 3) giving vertex coordinates. + faces: LongTensor of shape (F, 3) giving faces. + verts_normals: FloatTensor of shape (V, 3) giving vertex normals. + verts_colors: FloatTensor of shape (V, 3) giving vertex colors. + ascii: (bool) whether to use the ascii ply format. + decimal_places: Number of decimal places for saving if ascii=True. + colors_as_uint8: Whether to save colors as numbers in the range + [0, 255] instead of float32. + """ + _write_ply_header( + f, + verts=verts, + faces=faces, + verts_normals=verts_normals, + verts_colors=verts_colors, + ascii=ascii, + colors_as_uint8=colors_as_uint8, + ) + + if not (len(verts)): + warnings.warn("Empty 'verts' provided") + return + + color_np_type = np.ubyte if colors_as_uint8 else np.float32 + verts_dtype = [("verts", np.float32, 3)] + if verts_normals is not None: + verts_dtype.append(("normals", np.float32, 3)) + if verts_colors is not None: + verts_dtype.append(("colors", color_np_type, 3)) + + vert_data = np.zeros(verts.shape[0], dtype=verts_dtype) + vert_data["verts"] = verts.detach().cpu().numpy() + if verts_normals is not None: + vert_data["normals"] = verts_normals.detach().cpu().numpy() + if verts_colors is not None: + color_data = verts_colors.detach().cpu().numpy() + if colors_as_uint8: + vert_data["colors"] = np.rint(color_data * 255) + else: + vert_data["colors"] = color_data + + if ascii: + if decimal_places is None: + float_str = b"%f" + else: + float_str = b"%" + b".%df" % decimal_places + float_group_str = (float_str + b" ") * 3 + formats = [float_group_str] + if verts_normals is not None: + formats.append(float_group_str) + if verts_colors is not None: + formats.append(b"%d %d %d " if colors_as_uint8 else float_group_str) + formats[-1] = formats[-1][:-1] + b"\n" + for line_data in vert_data: + for data, format in zip(line_data, formats): + f.write(format % tuple(data)) + else: + if isinstance(f, BytesIO): + # tofile only works with real files, but is faster than this. + f.write(vert_data.tobytes()) + else: + vert_data.tofile(f) + + if faces is not None: + faces_array = faces.detach().cpu().numpy() + + _check_faces_indices(faces, max_index=verts.shape[0]) + + if len(faces_array): + if ascii: + np.savetxt(f, faces_array, "3 %d %d %d") + else: + faces_recs = np.zeros( + len(faces_array), + dtype=[("count", np.uint8), ("vertex_indices", np.uint32, 3)], + ) + faces_recs["count"] = 3 + faces_recs["vertex_indices"] = faces_array + faces_uints = faces_recs.view(np.uint8) + + if isinstance(f, BytesIO): + f.write(faces_uints.tobytes()) + else: + faces_uints.tofile(f) + + +def save_ply( + f, + verts: torch.Tensor, + faces: Optional[torch.LongTensor] = None, + verts_normals: Optional[torch.Tensor] = None, + ascii: bool = False, + decimal_places: Optional[int] = None, + path_manager: Optional[PathManager] = None, +) -> None: + """ + Save a mesh to a .ply file. + + Args: + f: File (or path) to which the mesh should be written. + verts: FloatTensor of shape (V, 3) giving vertex coordinates. + faces: LongTensor of shape (F, 3) giving faces. + verts_normals: FloatTensor of shape (V, 3) giving vertex normals. + ascii: (bool) whether to use the ascii ply format. + decimal_places: Number of decimal places for saving if ascii=True. + path_manager: PathManager for interpreting f if it is a str. + """ + + if len(verts) and not (verts.dim() == 2 and verts.size(1) == 3): + message = "Argument 'verts' should either be empty or of shape (num_verts, 3)." + raise ValueError(message) + + if ( + faces is not None + and len(faces) + and not (faces.dim() == 2 and faces.size(1) == 3) + ): + message = "Argument 'faces' should either be empty or of shape (num_faces, 3)." + raise ValueError(message) + + if ( + verts_normals is not None + and len(verts_normals) + and not ( + verts_normals.dim() == 2 + and verts_normals.size(1) == 3 + and verts_normals.size(0) == verts.size(0) + ) + ): + message = "Argument 'verts_normals' should either be empty or of shape (num_verts, 3)." + raise ValueError(message) + + if path_manager is None: + path_manager = PathManager() + with _open_file(f, path_manager, "wb") as f: + _save_ply( + f, + verts=verts, + faces=faces, + verts_normals=verts_normals, + verts_colors=None, + ascii=ascii, + decimal_places=decimal_places, + colors_as_uint8=False, + ) + + +class MeshPlyFormat(MeshFormatInterpreter): + def __init__(self) -> None: + self.known_suffixes = (".ply",) + + def read( + self, + path: PathOrStr, + include_textures: bool, + device, + path_manager: PathManager, + **kwargs, + ) -> Optional[Meshes]: + if not endswith(path, self.known_suffixes): + return None + + data = _load_ply(f=path, path_manager=path_manager) + faces = data.faces + if faces is None: + faces = torch.zeros(0, 3, dtype=torch.int64) + + texture = None + if include_textures: + if data.verts_colors is not None: + texture = TexturesVertex([data.verts_colors.to(device)]) + elif data.verts_texture_uvs is not None: + texture_file_path = None + for comment in data.header.comments: + if "TextureFile" in comment: + given_texture_file = comment.split(" ")[-1] + texture_file_path = os.path.join( + os.path.dirname(str(path)), given_texture_file + ) + if texture_file_path is not None: + texture_map = _read_image( + texture_file_path, path_manager, format="RGB" + ) + texture_map = torch.tensor(texture_map, dtype=torch.float32) / 255.0 + texture = TexturesUV( + [texture_map.to(device)], + [faces.to(device)], + [data.verts_texture_uvs.to(device)], + ) + + verts_normals = None + if data.verts_normals is not None: + verts_normals = [data.verts_normals.to(device)] + mesh = Meshes( + verts=[data.verts.to(device)], + faces=[faces.to(device)], + textures=texture, + verts_normals=verts_normals, + ) + return mesh + + def save( + self, + data: Meshes, + path: PathOrStr, + path_manager: PathManager, + binary: Optional[bool], + decimal_places: Optional[int] = None, + colors_as_uint8: bool = False, + **kwargs, + ) -> bool: + """ + Extra optional args: + colors_as_uint8: (bool) Whether to save colors as numbers in the + range [0, 255] instead of float32. + """ + if not endswith(path, self.known_suffixes): + return False + + verts = data.verts_list()[0] + faces = data.faces_list()[0] + + if data.has_verts_normals(): + verts_normals = data.verts_normals_list()[0] + else: + verts_normals = None + + if isinstance(data.textures, TexturesVertex): + mesh_verts_colors = data.textures.verts_features_list()[0] + n_colors = mesh_verts_colors.shape[1] + if n_colors == 3: + verts_colors = mesh_verts_colors + else: + warnings.warn( + f"Texture will not be saved as it has {n_colors} colors, not 3." + ) + verts_colors = None + else: + verts_colors = None + + with _open_file(path, path_manager, "wb") as f: + _save_ply( + f=f, + verts=verts, + faces=faces, + verts_colors=verts_colors, + verts_normals=verts_normals, + ascii=binary is False, + decimal_places=decimal_places, + colors_as_uint8=colors_as_uint8, + ) + return True + + +class PointcloudPlyFormat(PointcloudFormatInterpreter): + def __init__(self) -> None: + self.known_suffixes = (".ply",) + + def read( + self, + path: PathOrStr, + device, + path_manager: PathManager, + **kwargs, + ) -> Optional[Pointclouds]: + if not endswith(path, self.known_suffixes): + return None + + data = _load_ply(f=path, path_manager=path_manager) + features = None + if data.verts_colors is not None: + features = [data.verts_colors.to(device)] + normals = None + if data.verts_normals is not None: + normals = [data.verts_normals.to(device)] + + pointcloud = Pointclouds( + points=[data.verts.to(device)], features=features, normals=normals + ) + return pointcloud + + def save( + self, + data: Pointclouds, + path: PathOrStr, + path_manager: PathManager, + binary: Optional[bool], + decimal_places: Optional[int] = None, + colors_as_uint8: bool = False, + **kwargs, + ) -> bool: + """ + Extra optional args: + colors_as_uint8: (bool) Whether to save colors as numbers in the + range [0, 255] instead of float32. + """ + if not endswith(path, self.known_suffixes): + return False + + points = data.points_list()[0] + features = data.features_packed() + normals = data.normals_packed() + + with _open_file(path, path_manager, "wb") as f: + _save_ply( + f=f, + verts=points, + verts_colors=features, + verts_normals=normals, + faces=None, + ascii=binary is False, + decimal_places=decimal_places, + colors_as_uint8=colors_as_uint8, + ) + return True diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/utils.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..467c7c14b5f76a23090a5f3375fe5948fa23a460 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/io/utils.py @@ -0,0 +1,85 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import contextlib +import pathlib +import warnings +from typing import cast, ContextManager, IO, Optional, Union + +import numpy as np +import torch +from iopath.common.file_io import PathManager +from PIL import Image + +from ..common.datatypes import Device + + +PathOrStr = Union[pathlib.Path, str] + + +def _open_file(f, path_manager: PathManager, mode: str = "r") -> ContextManager[IO]: + if isinstance(f, str): + # pyre-fixme[6]: For 2nd argument expected `Union[typing_extensions.Literal['... + f = path_manager.open(f, mode) + return contextlib.closing(f) + elif isinstance(f, pathlib.Path): + f = f.open(mode) + return contextlib.closing(f) + else: + return contextlib.nullcontext(cast(IO, f)) + + +def _make_tensor( + data, cols: int, dtype: torch.dtype, device: Device = "cpu" +) -> torch.Tensor: + """ + Return a 2D tensor with the specified cols and dtype filled with data, + even when data is empty. + """ + if not len(data): + return torch.zeros((0, cols), dtype=dtype, device=device) + + return torch.tensor(data, dtype=dtype, device=device) + + +def _check_faces_indices( + faces_indices: torch.Tensor, max_index: int, pad_value: Optional[int] = None +) -> torch.Tensor: + if pad_value is None: + mask = torch.ones(faces_indices.shape[:-1]).bool() # Keep all faces + else: + mask = faces_indices.ne(pad_value).any(dim=-1) + if torch.any(faces_indices[mask] >= max_index) or torch.any( + faces_indices[mask] < 0 + ): + warnings.warn("Faces have invalid indices") + return faces_indices + + +def _read_image(file_name: str, path_manager: PathManager, format=None): + """ + Read an image from a file using Pillow. + Args: + file_name: image file path. + path_manager: PathManager for interpreting file_name. + format: one of ["RGB", "BGR"] + Returns: + image: an image of shape (H, W, C). + """ + if format not in ["RGB", "BGR"]: + raise ValueError("format can only be one of [RGB, BGR]; got %s", format) + with path_manager.open(file_name, "rb") as f: + image = Image.open(f) + if format is not None: + # PIL only supports RGB. First convert to RGB and flip channels + # below for BGR. + image = image.convert("RGB") + image = np.asarray(image).astype(np.float32) + if format == "BGR": + image = image[:, :, ::-1] + return image diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/loss/__init__.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/loss/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..94b012c45290aa772c9905b4cc7d1c80d1e8736a --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/loss/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +from .chamfer import chamfer_distance + +from .mesh_edge_loss import mesh_edge_loss + +from .mesh_laplacian_smoothing import mesh_laplacian_smoothing + +from .mesh_normal_consistency import mesh_normal_consistency +from .point_mesh_distance import point_mesh_edge_distance, point_mesh_face_distance + + +__all__ = [k for k in globals().keys() if not k.startswith("_")] diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/loss/chamfer.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/loss/chamfer.py new file mode 100644 index 0000000000000000000000000000000000000000..67289020cde5b59f4fbba40dda8c35efb57f0354 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/loss/chamfer.py @@ -0,0 +1,270 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from typing import Union + +import torch +import torch.nn.functional as F +from pytorch3d.ops.knn import knn_gather, knn_points +from pytorch3d.structures.pointclouds import Pointclouds + + +def _validate_chamfer_reduction_inputs( + batch_reduction: Union[str, None], point_reduction: Union[str, None] +) -> None: + """Check the requested reductions are valid. + + Args: + batch_reduction: Reduction operation to apply for the loss across the + batch, can be one of ["mean", "sum"] or None. + point_reduction: Reduction operation to apply for the loss across the + points, can be one of ["mean", "sum"] or None. + """ + if batch_reduction is not None and batch_reduction not in ["mean", "sum"]: + raise ValueError('batch_reduction must be one of ["mean", "sum"] or None') + if point_reduction is not None and point_reduction not in ["mean", "sum"]: + raise ValueError('point_reduction must be one of ["mean", "sum"] or None') + if point_reduction is None and batch_reduction is not None: + raise ValueError("Batch reduction must be None if point_reduction is None") + + +def _handle_pointcloud_input( + points: Union[torch.Tensor, Pointclouds], + lengths: Union[torch.Tensor, None], + normals: Union[torch.Tensor, None], +): + """ + If points is an instance of Pointclouds, retrieve the padded points tensor + along with the number of points per batch and the padded normals. + Otherwise, return the input points (and normals) with the number of points per cloud + set to the size of the second dimension of `points`. + """ + if isinstance(points, Pointclouds): + X = points.points_padded() + lengths = points.num_points_per_cloud() + normals = points.normals_padded() # either a tensor or None + elif torch.is_tensor(points): + if points.ndim != 3: + raise ValueError("Expected points to be of shape (N, P, D)") + X = points + if lengths is not None: + if lengths.ndim != 1 or lengths.shape[0] != X.shape[0]: + raise ValueError("Expected lengths to be of shape (N,)") + if lengths.max() > X.shape[1]: + raise ValueError("A length value was too long") + if lengths is None: + lengths = torch.full( + (X.shape[0],), X.shape[1], dtype=torch.int64, device=points.device + ) + if normals is not None and normals.ndim != 3: + raise ValueError("Expected normals to be of shape (N, P, 3") + else: + raise ValueError( + "The input pointclouds should be either " + + "Pointclouds objects or torch.Tensor of shape " + + "(minibatch, num_points, 3)." + ) + return X, lengths, normals + + +def _chamfer_distance_single_direction( + x, + y, + x_lengths, + y_lengths, + x_normals, + y_normals, + weights, + batch_reduction: Union[str, None], + point_reduction: Union[str, None], + norm: int, + abs_cosine: bool, +): + return_normals = x_normals is not None and y_normals is not None + + N, P1, D = x.shape + + # Check if inputs are heterogeneous and create a lengths mask. + is_x_heterogeneous = (x_lengths != P1).any() + x_mask = ( + torch.arange(P1, device=x.device)[None] >= x_lengths[:, None] + ) # shape [N, P1] + if y.shape[0] != N or y.shape[2] != D: + raise ValueError("y does not have the correct shape.") + if weights is not None: + if weights.size(0) != N: + raise ValueError("weights must be of shape (N,).") + if not (weights >= 0).all(): + raise ValueError("weights cannot be negative.") + if weights.sum() == 0.0: + weights = weights.view(N, 1) + if batch_reduction in ["mean", "sum"]: + return ( + (x.sum((1, 2)) * weights).sum() * 0.0, + (x.sum((1, 2)) * weights).sum() * 0.0, + ) + return ((x.sum((1, 2)) * weights) * 0.0, (x.sum((1, 2)) * weights) * 0.0) + + cham_norm_x = x.new_zeros(()) + + x_nn = knn_points(x, y, lengths1=x_lengths, lengths2=y_lengths, norm=norm, K=1) + cham_x = x_nn.dists[..., 0] # (N, P1) + + if is_x_heterogeneous: + cham_x[x_mask] = 0.0 + + if weights is not None: + cham_x *= weights.view(N, 1) + + if return_normals: + # Gather the normals using the indices and keep only value for k=0 + x_normals_near = knn_gather(y_normals, x_nn.idx, y_lengths)[..., 0, :] + + cosine_sim = F.cosine_similarity(x_normals, x_normals_near, dim=2, eps=1e-6) + # If abs_cosine, ignore orientation and take the absolute value of the cosine sim. + cham_norm_x = 1 - (torch.abs(cosine_sim) if abs_cosine else cosine_sim) + + if is_x_heterogeneous: + cham_norm_x[x_mask] = 0.0 + + if weights is not None: + cham_norm_x *= weights.view(N, 1) + + if point_reduction is not None: + # Apply point reduction + cham_x = cham_x.sum(1) # (N,) + if return_normals: + cham_norm_x = cham_norm_x.sum(1) # (N,) + if point_reduction == "mean": + x_lengths_clamped = x_lengths.clamp(min=1) + cham_x /= x_lengths_clamped + if return_normals: + cham_norm_x /= x_lengths_clamped + + if batch_reduction is not None: + # batch_reduction == "sum" + cham_x = cham_x.sum() + if return_normals: + cham_norm_x = cham_norm_x.sum() + if batch_reduction == "mean": + div = weights.sum() if weights is not None else max(N, 1) + cham_x /= div + if return_normals: + cham_norm_x /= div + + cham_dist = cham_x + cham_normals = cham_norm_x if return_normals else None + return cham_dist, cham_normals + + +def chamfer_distance( + x, + y, + x_lengths=None, + y_lengths=None, + x_normals=None, + y_normals=None, + weights=None, + batch_reduction: Union[str, None] = "mean", + point_reduction: Union[str, None] = "mean", + norm: int = 2, + single_directional: bool = False, + abs_cosine: bool = True, +): + """ + Chamfer distance between two pointclouds x and y. + + Args: + x: FloatTensor of shape (N, P1, D) or a Pointclouds object representing + a batch of point clouds with at most P1 points in each batch element, + batch size N and feature dimension D. + y: FloatTensor of shape (N, P2, D) or a Pointclouds object representing + a batch of point clouds with at most P2 points in each batch element, + batch size N and feature dimension D. + x_lengths: Optional LongTensor of shape (N,) giving the number of points in each + cloud in x. + y_lengths: Optional LongTensor of shape (N,) giving the number of points in each + cloud in y. + x_normals: Optional FloatTensor of shape (N, P1, D). + y_normals: Optional FloatTensor of shape (N, P2, D). + weights: Optional FloatTensor of shape (N,) giving weights for + batch elements for reduction operation. + batch_reduction: Reduction operation to apply for the loss across the + batch, can be one of ["mean", "sum"] or None. + point_reduction: Reduction operation to apply for the loss across the + points, can be one of ["mean", "sum"] or None. + norm: int indicates the norm used for the distance. Supports 1 for L1 and 2 for L2. + single_directional: If False (default), loss comes from both the distance between + each point in x and its nearest neighbor in y and each point in y and its nearest + neighbor in x. If True, loss is the distance between each point in x and its + nearest neighbor in y. + abs_cosine: If False, loss_normals is from one minus the cosine similarity. + If True (default), loss_normals is from one minus the absolute value of the + cosine similarity, which means that exactly opposite normals are considered + equivalent to exactly matching normals, i.e. sign does not matter. + + Returns: + 2-element tuple containing + + - **loss**: Tensor giving the reduced distance between the pointclouds + in x and the pointclouds in y. If point_reduction is None, a 2-element + tuple of Tensors containing forward and backward loss terms shaped (N, P1) + and (N, P2) (if single_directional is False) or a Tensor containing loss + terms shaped (N, P1) (if single_directional is True) is returned. + - **loss_normals**: Tensor giving the reduced cosine distance of normals + between pointclouds in x and pointclouds in y. Returns None if + x_normals and y_normals are None. If point_reduction is None, a 2-element + tuple of Tensors containing forward and backward loss terms shaped (N, P1) + and (N, P2) (if single_directional is False) or a Tensor containing loss + terms shaped (N, P1) (if single_directional is True) is returned. + """ + _validate_chamfer_reduction_inputs(batch_reduction, point_reduction) + + if not ((norm == 1) or (norm == 2)): + raise ValueError("Support for 1 or 2 norm.") + x, x_lengths, x_normals = _handle_pointcloud_input(x, x_lengths, x_normals) + y, y_lengths, y_normals = _handle_pointcloud_input(y, y_lengths, y_normals) + + cham_x, cham_norm_x = _chamfer_distance_single_direction( + x, + y, + x_lengths, + y_lengths, + x_normals, + y_normals, + weights, + batch_reduction, + point_reduction, + norm, + abs_cosine, + ) + if single_directional: + return cham_x, cham_norm_x + else: + cham_y, cham_norm_y = _chamfer_distance_single_direction( + y, + x, + y_lengths, + x_lengths, + y_normals, + x_normals, + weights, + batch_reduction, + point_reduction, + norm, + abs_cosine, + ) + if point_reduction is not None: + return ( + cham_x + cham_y, + (cham_norm_x + cham_norm_y) if cham_norm_x is not None else None, + ) + return ( + (cham_x, cham_y), + (cham_norm_x, cham_norm_y) if cham_norm_x is not None else None, + ) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/loss/mesh_edge_loss.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/loss/mesh_edge_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..615fe5072b4411405c0f8e3fbf1b78ea9814d6c9 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/loss/mesh_edge_loss.py @@ -0,0 +1,52 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import torch + + +def mesh_edge_loss(meshes, target_length: float = 0.0): + """ + Computes mesh edge length regularization loss averaged across all meshes + in a batch. Each mesh contributes equally to the final loss, regardless of + the number of edges per mesh in the batch by weighting each mesh with the + inverse number of edges. For example, if mesh 3 (out of N) has only E=4 + edges, then the loss for each edge in mesh 3 should be multiplied by 1/E to + contribute to the final loss. + + Args: + meshes: Meshes object with a batch of meshes. + target_length: Resting value for the edge length. + + Returns: + loss: Average loss across the batch. Returns 0 if meshes contains + no meshes or all empty meshes. + """ + if meshes.isempty(): + return torch.tensor( + [0.0], dtype=torch.float32, device=meshes.device, requires_grad=True + ) + + N = len(meshes) + edges_packed = meshes.edges_packed() # (sum(E_n), 3) + verts_packed = meshes.verts_packed() # (sum(V_n), 3) + edge_to_mesh_idx = meshes.edges_packed_to_mesh_idx() # (sum(E_n), ) + num_edges_per_mesh = meshes.num_edges_per_mesh() # N + + # Determine the weight for each edge based on the number of edges in the + # mesh it corresponds to. + # TODO (nikhilar) Find a faster way of computing the weights for each edge + # as this is currently a bottleneck for meshes with a large number of faces. + weights = num_edges_per_mesh.gather(0, edge_to_mesh_idx) + weights = 1.0 / weights.float() + + verts_edges = verts_packed[edges_packed] + v0, v1 = verts_edges.unbind(1) + loss = ((v0 - v1).norm(dim=1, p=2) - target_length) ** 2.0 + loss = loss * weights + + return loss.sum() / N diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/loss/mesh_laplacian_smoothing.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/loss/mesh_laplacian_smoothing.py new file mode 100644 index 0000000000000000000000000000000000000000..02037841d74f849c026e98192be3275be20522d7 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/loss/mesh_laplacian_smoothing.py @@ -0,0 +1,137 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +import torch +from pytorch3d.ops import cot_laplacian + + +def mesh_laplacian_smoothing(meshes, method: str = "uniform"): + r""" + Computes the laplacian smoothing objective for a batch of meshes. + This function supports three variants of Laplacian smoothing, + namely with uniform weights("uniform"), with cotangent weights ("cot"), + and cotangent curvature ("cotcurv").For more details read [1, 2]. + + Args: + meshes: Meshes object with a batch of meshes. + method: str specifying the method for the laplacian. + Returns: + loss: Average laplacian smoothing loss across the batch. + Returns 0 if meshes contains no meshes or all empty meshes. + + Consider a mesh M = (V, F), with verts of shape Nx3 and faces of shape Mx3. + The Laplacian matrix L is a NxN tensor such that LV gives a tensor of vectors: + for a uniform Laplacian, LuV[i] points to the centroid of its neighboring + vertices, a cotangent Laplacian LcV[i] is known to be an approximation of + the surface normal, while the curvature variant LckV[i] scales the normals + by the discrete mean curvature. For vertex i, assume S[i] is the set of + neighboring vertices to i, a_ij and b_ij are the "outside" angles in the + two triangles connecting vertex v_i and its neighboring vertex v_j + for j in S[i], as seen in the diagram below. + + .. code-block:: python + + a_ij + /\ + / \ + / \ + / \ + v_i /________\ v_j + \ / + \ / + \ / + \ / + \/ + b_ij + + The definition of the Laplacian is LV[i] = sum_j w_ij (v_j - v_i) + For the uniform variant, w_ij = 1 / |S[i]| + For the cotangent variant, + w_ij = (cot a_ij + cot b_ij) / (sum_k cot a_ik + cot b_ik) + For the cotangent curvature, w_ij = (cot a_ij + cot b_ij) / (4 A[i]) + where A[i] is the sum of the areas of all triangles containing vertex v_i. + + There is a nice trigonometry identity to compute cotangents. Consider a triangle + with side lengths A, B, C and angles a, b, c. + + .. code-block:: python + + c + /|\ + / | \ + / | \ + B / H| \ A + / | \ + / | \ + /a_____|_____b\ + C + + Then cot a = (B^2 + C^2 - A^2) / 4 * area + We know that area = CH/2, and by the law of cosines we have + + A^2 = B^2 + C^2 - 2BC cos a => B^2 + C^2 - A^2 = 2BC cos a + + Putting these together, we get: + + B^2 + C^2 - A^2 2BC cos a + _______________ = _________ = (B/H) cos a = cos a / sin a = cot a + 4 * area 2CH + + + [1] Desbrun et al, "Implicit fairing of irregular meshes using diffusion + and curvature flow", SIGGRAPH 1999. + + [2] Nealan et al, "Laplacian Mesh Optimization", Graphite 2006. + """ + + if meshes.isempty(): + return torch.tensor( + [0.0], dtype=torch.float32, device=meshes.device, requires_grad=True + ) + + N = len(meshes) + verts_packed = meshes.verts_packed() # (sum(V_n), 3) + faces_packed = meshes.faces_packed() # (sum(F_n), 3) + num_verts_per_mesh = meshes.num_verts_per_mesh() # (N,) + verts_packed_idx = meshes.verts_packed_to_mesh_idx() # (sum(V_n),) + weights = num_verts_per_mesh.gather(0, verts_packed_idx) # (sum(V_n),) + weights = 1.0 / weights.float() + + # We don't want to backprop through the computation of the Laplacian; + # just treat it as a magic constant matrix that is used to transform + # verts into normals + with torch.no_grad(): + if method == "uniform": + L = meshes.laplacian_packed() + elif method in ["cot", "cotcurv"]: + L, inv_areas = cot_laplacian(verts_packed, faces_packed) + if method == "cot": + norm_w = torch.sparse.sum(L, dim=1).to_dense().view(-1, 1) + idx = norm_w > 0 + # pyre-fixme[58]: `/` is not supported for operand types `float` and + # `Tensor`. + norm_w[idx] = 1.0 / norm_w[idx] + else: + L_sum = torch.sparse.sum(L, dim=1).to_dense().view(-1, 1) + norm_w = 0.25 * inv_areas + else: + raise ValueError("Method should be one of {uniform, cot, cotcurv}") + + if method == "uniform": + loss = L.mm(verts_packed) + elif method == "cot": + # pyre-fixme[61]: `norm_w` is undefined, or not always defined. + loss = L.mm(verts_packed) * norm_w - verts_packed + elif method == "cotcurv": + # pyre-fixme[61]: `norm_w` may not be initialized here. + loss = (L.mm(verts_packed) - L_sum * verts_packed) * norm_w + loss = loss.norm(dim=1) + + loss = loss * weights + return loss.sum() / N diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/loss/mesh_normal_consistency.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/loss/mesh_normal_consistency.py new file mode 100644 index 0000000000000000000000000000000000000000..5466df30d19a62bd90984acdb91847f3c412eeab --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/loss/mesh_normal_consistency.py @@ -0,0 +1,134 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import torch +from pytorch3d import _C + + +def mesh_normal_consistency(meshes): + r""" + Computes the normal consistency of each mesh in meshes. + We compute the normal consistency for each pair of neighboring faces. + If e = (v0, v1) is the connecting edge of two neighboring faces f0 and f1, + then the normal consistency between f0 and f1 + + .. code-block:: python + + a + /\ + / \ + / f0 \ + / \ + v0 /____e___\ v1 + \ / + \ / + \ f1 / + \ / + \/ + b + + The normal consistency is + + .. code-block:: python + + nc(f0, f1) = 1 - cos(n0, n1) + + where cos(n0, n1) = n0^n1 / ||n0|| / ||n1|| is the cosine of the angle + between the normals n0 and n1, and + + n0 = (v1 - v0) x (a - v0) + n1 = - (v1 - v0) x (b - v0) = (b - v0) x (v1 - v0) + + This means that if nc(f0, f1) = 0 then n0 and n1 point to the same + direction, while if nc(f0, f1) = 2 then n0 and n1 point opposite direction. + + .. note:: + For well-constructed meshes the assumption that only two faces share an + edge is true. This assumption could make the implementation easier and faster. + This implementation does not follow this assumption. All the faces sharing e, + which can be any in number, are discovered. + + Args: + meshes: Meshes object with a batch of meshes. + + Returns: + loss: Average normal consistency across the batch. + Returns 0 if meshes contains no meshes or all empty meshes. + """ + if meshes.isempty(): + return torch.tensor( + [0.0], dtype=torch.float32, device=meshes.device, requires_grad=True + ) + + N = len(meshes) + verts_packed = meshes.verts_packed() # (sum(V_n), 3) + faces_packed = meshes.faces_packed() # (sum(F_n), 3) + edges_packed = meshes.edges_packed() # (sum(E_n), 2) + verts_packed_to_mesh_idx = meshes.verts_packed_to_mesh_idx() # (sum(V_n),) + face_to_edge = meshes.faces_packed_to_edges_packed() # (sum(F_n), 3) + E = edges_packed.shape[0] # sum(E_n) + F = faces_packed.shape[0] # sum(F_n) + + # We don't want gradients for the following operation. The goal is to + # find for each edge e all the vertices associated with e. In the example + # above, the vertices associated with e are (a, b), i.e. the points connected + # on faces to e. + with torch.no_grad(): + edge_idx = face_to_edge.reshape(F * 3) # (3 * F,) indexes into edges + vert_idx = ( + faces_packed.view(1, F, 3).expand(3, F, 3).transpose(0, 1).reshape(3 * F, 3) + ) + edge_idx, edge_sort_idx = edge_idx.sort() + vert_idx = vert_idx[edge_sort_idx] + + # In well constructed meshes each edge is shared by precisely 2 faces + # However, in many meshes, this assumption is not always satisfied. + # We want to find all faces that share an edge, a number which can + # vary and which depends on the topology. + # In particular, we find the vertices not on the edge on the shared faces. + # In the example above, we want to associate edge e with vertices a and b. + # This operation is done more efficiently in cpu with lists. + # TODO(gkioxari) find a better way to do this. + + # edge_idx represents the index of the edge for each vertex. We can count + # the number of vertices which are associated with each edge. + # There can be a different number for each edge. + edge_num = edge_idx.bincount(minlength=E) + + # This calculates all pairs of vertices which are opposite to the same edge. + vert_edge_pair_idx = _C.mesh_normal_consistency_find_verts(edge_num.cpu()).to( + edge_num.device + ) + + if vert_edge_pair_idx.shape[0] == 0: + return torch.tensor( + [0.0], dtype=torch.float32, device=meshes.device, requires_grad=True + ) + + v0_idx = edges_packed[edge_idx, 0] + v0 = verts_packed[v0_idx] + v1_idx = edges_packed[edge_idx, 1] + v1 = verts_packed[v1_idx] + + # two of the following cross products are zeros as they are cross product + # with either (v1-v0)x(v1-v0) or (v1-v0)x(v0-v0) + n_temp0 = (v1 - v0).cross(verts_packed[vert_idx[:, 0]] - v0, dim=1) + n_temp1 = (v1 - v0).cross(verts_packed[vert_idx[:, 1]] - v0, dim=1) + n_temp2 = (v1 - v0).cross(verts_packed[vert_idx[:, 2]] - v0, dim=1) + n = n_temp0 + n_temp1 + n_temp2 + n0 = n[vert_edge_pair_idx[:, 0]] + n1 = -n[vert_edge_pair_idx[:, 1]] + loss = 1 - torch.cosine_similarity(n0, n1, dim=1) + + verts_packed_to_mesh_idx = verts_packed_to_mesh_idx[vert_idx[:, 0]] + verts_packed_to_mesh_idx = verts_packed_to_mesh_idx[vert_edge_pair_idx[:, 0]] + num_normals = verts_packed_to_mesh_idx.bincount(minlength=N) + weights = 1.0 / num_normals[verts_packed_to_mesh_idx].float() + + loss = loss * weights + return loss.sum() / N diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/loss/point_mesh_distance.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/loss/point_mesh_distance.py new file mode 100644 index 0000000000000000000000000000000000000000..b30cd05e88544c2b255107bae6001c12fbfdf307 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/loss/point_mesh_distance.py @@ -0,0 +1,398 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from pytorch3d import _C +from pytorch3d.structures import Meshes, Pointclouds +from torch.autograd import Function +from torch.autograd.function import once_differentiable + + +""" +This file defines distances between meshes and pointclouds. +The functions make use of the definition of a distance between a point and +an edge segment or the distance of a point and a triangle (face). + +The exact mathematical formulations and implementations of these +distances can be found in `csrc/utils/geometry_utils.cuh`. +""" + +_DEFAULT_MIN_TRIANGLE_AREA: float = 5e-3 + + +# PointFaceDistance +class _PointFaceDistance(Function): + """ + Torch autograd Function wrapper PointFaceDistance Cuda implementation + """ + + @staticmethod + def forward( + ctx, + points, + points_first_idx, + tris, + tris_first_idx, + max_points, + min_triangle_area=_DEFAULT_MIN_TRIANGLE_AREA, + ): + """ + Args: + ctx: Context object used to calculate gradients. + points: FloatTensor of shape `(P, 3)` + points_first_idx: LongTensor of shape `(N,)` indicating the first point + index in each example in the batch + tris: FloatTensor of shape `(T, 3, 3)` of triangular faces. The `t`-th + triangular face is spanned by `(tris[t, 0], tris[t, 1], tris[t, 2])` + tris_first_idx: LongTensor of shape `(N,)` indicating the first face + index in each example in the batch + max_points: Scalar equal to maximum number of points in the batch + min_triangle_area: (float, defaulted) Triangles of area less than this + will be treated as points/lines. + Returns: + dists: FloatTensor of shape `(P,)`, where `dists[p]` is the squared + euclidean distance of `p`-th point to the closest triangular face + in the corresponding example in the batch + idxs: LongTensor of shape `(P,)` indicating the closest triangular face + in the corresponding example in the batch. + + `dists[p]` is + `d(points[p], tris[idxs[p], 0], tris[idxs[p], 1], tris[idxs[p], 2])` + where `d(u, v0, v1, v2)` is the distance of point `u` from the triangular + face `(v0, v1, v2)` + + """ + dists, idxs = _C.point_face_dist_forward( + points, + points_first_idx, + tris, + tris_first_idx, + max_points, + min_triangle_area, + ) + ctx.save_for_backward(points, tris, idxs) + ctx.min_triangle_area = min_triangle_area + return dists + + @staticmethod + @once_differentiable + def backward(ctx, grad_dists): + grad_dists = grad_dists.contiguous() + points, tris, idxs = ctx.saved_tensors + min_triangle_area = ctx.min_triangle_area + grad_points, grad_tris = _C.point_face_dist_backward( + points, tris, idxs, grad_dists, min_triangle_area + ) + return grad_points, None, grad_tris, None, None, None + + +point_face_distance = _PointFaceDistance.apply + + +# FacePointDistance +class _FacePointDistance(Function): + """ + Torch autograd Function wrapper FacePointDistance Cuda implementation + """ + + @staticmethod + def forward( + ctx, + points, + points_first_idx, + tris, + tris_first_idx, + max_tris, + min_triangle_area=_DEFAULT_MIN_TRIANGLE_AREA, + ): + """ + Args: + ctx: Context object used to calculate gradients. + points: FloatTensor of shape `(P, 3)` + points_first_idx: LongTensor of shape `(N,)` indicating the first point + index in each example in the batch + tris: FloatTensor of shape `(T, 3, 3)` of triangular faces. The `t`-th + triangular face is spanned by `(tris[t, 0], tris[t, 1], tris[t, 2])` + tris_first_idx: LongTensor of shape `(N,)` indicating the first face + index in each example in the batch + max_tris: Scalar equal to maximum number of faces in the batch + min_triangle_area: (float, defaulted) Triangles of area less than this + will be treated as points/lines. + Returns: + dists: FloatTensor of shape `(T,)`, where `dists[t]` is the squared + euclidean distance of `t`-th triangular face to the closest point in the + corresponding example in the batch + idxs: LongTensor of shape `(T,)` indicating the closest point in the + corresponding example in the batch. + + `dists[t] = d(points[idxs[t]], tris[t, 0], tris[t, 1], tris[t, 2])`, + where `d(u, v0, v1, v2)` is the distance of point `u` from the triangular + face `(v0, v1, v2)`. + """ + dists, idxs = _C.face_point_dist_forward( + points, points_first_idx, tris, tris_first_idx, max_tris, min_triangle_area + ) + ctx.save_for_backward(points, tris, idxs) + ctx.min_triangle_area = min_triangle_area + return dists + + @staticmethod + @once_differentiable + def backward(ctx, grad_dists): + grad_dists = grad_dists.contiguous() + points, tris, idxs = ctx.saved_tensors + min_triangle_area = ctx.min_triangle_area + grad_points, grad_tris = _C.face_point_dist_backward( + points, tris, idxs, grad_dists, min_triangle_area + ) + return grad_points, None, grad_tris, None, None, None + + +face_point_distance = _FacePointDistance.apply + + +# PointEdgeDistance +class _PointEdgeDistance(Function): + """ + Torch autograd Function wrapper PointEdgeDistance Cuda implementation + """ + + @staticmethod + def forward(ctx, points, points_first_idx, segms, segms_first_idx, max_points): + """ + Args: + ctx: Context object used to calculate gradients. + points: FloatTensor of shape `(P, 3)` + points_first_idx: LongTensor of shape `(N,)` indicating the first point + index for each example in the mesh + segms: FloatTensor of shape `(S, 2, 3)` of edge segments. The `s`-th + edge segment is spanned by `(segms[s, 0], segms[s, 1])` + segms_first_idx: LongTensor of shape `(N,)` indicating the first edge + index for each example in the mesh + max_points: Scalar equal to maximum number of points in the batch + Returns: + dists: FloatTensor of shape `(P,)`, where `dists[p]` is the squared + euclidean distance of `p`-th point to the closest edge in the + corresponding example in the batch + idxs: LongTensor of shape `(P,)` indicating the closest edge in the + corresponding example in the batch. + + `dists[p] = d(points[p], segms[idxs[p], 0], segms[idxs[p], 1])`, + where `d(u, v0, v1)` is the distance of point `u` from the edge segment + spanned by `(v0, v1)`. + """ + dists, idxs = _C.point_edge_dist_forward( + points, points_first_idx, segms, segms_first_idx, max_points + ) + ctx.save_for_backward(points, segms, idxs) + return dists + + @staticmethod + @once_differentiable + def backward(ctx, grad_dists): + grad_dists = grad_dists.contiguous() + points, segms, idxs = ctx.saved_tensors + grad_points, grad_segms = _C.point_edge_dist_backward( + points, segms, idxs, grad_dists + ) + return grad_points, None, grad_segms, None, None + + +point_edge_distance = _PointEdgeDistance.apply + + +# EdgePointDistance +class _EdgePointDistance(Function): + """ + Torch autograd Function wrapper EdgePointDistance Cuda implementation + """ + + @staticmethod + def forward(ctx, points, points_first_idx, segms, segms_first_idx, max_segms): + """ + Args: + ctx: Context object used to calculate gradients. + points: FloatTensor of shape `(P, 3)` + points_first_idx: LongTensor of shape `(N,)` indicating the first point + index for each example in the mesh + segms: FloatTensor of shape `(S, 2, 3)` of edge segments. The `s`-th + edge segment is spanned by `(segms[s, 0], segms[s, 1])` + segms_first_idx: LongTensor of shape `(N,)` indicating the first edge + index for each example in the mesh + max_segms: Scalar equal to maximum number of edges in the batch + Returns: + dists: FloatTensor of shape `(S,)`, where `dists[s]` is the squared + euclidean distance of `s`-th edge to the closest point in the + corresponding example in the batch + idxs: LongTensor of shape `(S,)` indicating the closest point in the + corresponding example in the batch. + + `dists[s] = d(points[idxs[s]], edges[s, 0], edges[s, 1])`, + where `d(u, v0, v1)` is the distance of point `u` from the segment + spanned by `(v0, v1)`. + """ + dists, idxs = _C.edge_point_dist_forward( + points, points_first_idx, segms, segms_first_idx, max_segms + ) + ctx.save_for_backward(points, segms, idxs) + return dists + + @staticmethod + @once_differentiable + def backward(ctx, grad_dists): + grad_dists = grad_dists.contiguous() + points, segms, idxs = ctx.saved_tensors + grad_points, grad_segms = _C.edge_point_dist_backward( + points, segms, idxs, grad_dists + ) + return grad_points, None, grad_segms, None, None + + +edge_point_distance = _EdgePointDistance.apply + + +def point_mesh_edge_distance(meshes: Meshes, pcls: Pointclouds): + """ + Computes the distance between a pointcloud and a mesh within a batch. + Given a pair `(mesh, pcl)` in the batch, we define the distance to be the + sum of two distances, namely `point_edge(mesh, pcl) + edge_point(mesh, pcl)` + + `point_edge(mesh, pcl)`: Computes the squared distance of each point p in pcl + to the closest edge segment in mesh and averages across all points in pcl + `edge_point(mesh, pcl)`: Computes the squared distance of each edge segment in mesh + to the closest point in pcl and averages across all edges in mesh. + + The above distance functions are applied for all `(mesh, pcl)` pairs in the batch + and then averaged across the batch. + + Args: + meshes: A Meshes data structure containing N meshes + pcls: A Pointclouds data structure containing N pointclouds + + Returns: + loss: The `point_edge(mesh, pcl) + edge_point(mesh, pcl)` distance + between all `(mesh, pcl)` in a batch averaged across the batch. + """ + if len(meshes) != len(pcls): + raise ValueError("meshes and pointclouds must be equal sized batches") + N = len(meshes) + + # packed representation for pointclouds + points = pcls.points_packed() # (P, 3) + points_first_idx = pcls.cloud_to_packed_first_idx() + max_points = pcls.num_points_per_cloud().max().item() + + # packed representation for edges + verts_packed = meshes.verts_packed() + edges_packed = meshes.edges_packed() + segms = verts_packed[edges_packed] # (S, 2, 3) + segms_first_idx = meshes.mesh_to_edges_packed_first_idx() + max_segms = meshes.num_edges_per_mesh().max().item() + + # point to edge distance: shape (P,) + point_to_edge = point_edge_distance( + points, points_first_idx, segms, segms_first_idx, max_points + ) + + # weight each example by the inverse of number of points in the example + point_to_cloud_idx = pcls.packed_to_cloud_idx() # (sum(P_i), ) + num_points_per_cloud = pcls.num_points_per_cloud() # (N,) + weights_p = num_points_per_cloud.gather(0, point_to_cloud_idx) + # pyre-fixme[58]: `/` is not supported for operand types `float` and `Tensor`. + weights_p = 1.0 / weights_p.float() + point_to_edge = point_to_edge * weights_p + point_dist = point_to_edge.sum() / N + + # edge to edge distance: shape (S,) + edge_to_point = edge_point_distance( + points, points_first_idx, segms, segms_first_idx, max_segms + ) + + # weight each example by the inverse of number of edges in the example + segm_to_mesh_idx = meshes.edges_packed_to_mesh_idx() # (sum(S_n),) + num_segms_per_mesh = meshes.num_edges_per_mesh() # (N,) + weights_s = num_segms_per_mesh.gather(0, segm_to_mesh_idx) + weights_s = 1.0 / weights_s.float() + edge_to_point = edge_to_point * weights_s + edge_dist = edge_to_point.sum() / N + + return point_dist + edge_dist + + +def point_mesh_face_distance( + meshes: Meshes, + pcls: Pointclouds, + min_triangle_area: float = _DEFAULT_MIN_TRIANGLE_AREA, +): + """ + Computes the distance between a pointcloud and a mesh within a batch. + Given a pair `(mesh, pcl)` in the batch, we define the distance to be the + sum of two distances, namely `point_face(mesh, pcl) + face_point(mesh, pcl)` + + `point_face(mesh, pcl)`: Computes the squared distance of each point p in pcl + to the closest triangular face in mesh and averages across all points in pcl + `face_point(mesh, pcl)`: Computes the squared distance of each triangular face in + mesh to the closest point in pcl and averages across all faces in mesh. + + The above distance functions are applied for all `(mesh, pcl)` pairs in the batch + and then averaged across the batch. + + Args: + meshes: A Meshes data structure containing N meshes + pcls: A Pointclouds data structure containing N pointclouds + min_triangle_area: (float, defaulted) Triangles of area less than this + will be treated as points/lines. + + Returns: + loss: The `point_face(mesh, pcl) + face_point(mesh, pcl)` distance + between all `(mesh, pcl)` in a batch averaged across the batch. + """ + + if len(meshes) != len(pcls): + raise ValueError("meshes and pointclouds must be equal sized batches") + N = len(meshes) + + # packed representation for pointclouds + points = pcls.points_packed() # (P, 3) + points_first_idx = pcls.cloud_to_packed_first_idx() + max_points = pcls.num_points_per_cloud().max().item() + + # packed representation for faces + verts_packed = meshes.verts_packed() + faces_packed = meshes.faces_packed() + tris = verts_packed[faces_packed] # (T, 3, 3) + tris_first_idx = meshes.mesh_to_faces_packed_first_idx() + max_tris = meshes.num_faces_per_mesh().max().item() + + # point to face distance: shape (P,) + point_to_face = point_face_distance( + points, points_first_idx, tris, tris_first_idx, max_points, min_triangle_area + ) + + # weight each example by the inverse of number of points in the example + point_to_cloud_idx = pcls.packed_to_cloud_idx() # (sum(P_i),) + num_points_per_cloud = pcls.num_points_per_cloud() # (N,) + weights_p = num_points_per_cloud.gather(0, point_to_cloud_idx) + # pyre-fixme[58]: `/` is not supported for operand types `float` and `Tensor`. + weights_p = 1.0 / weights_p.float() + point_to_face = point_to_face * weights_p + point_dist = point_to_face.sum() / N + + # face to point distance: shape (T,) + face_to_point = face_point_distance( + points, points_first_idx, tris, tris_first_idx, max_tris, min_triangle_area + ) + + # weight each example by the inverse of number of faces in the example + tri_to_mesh_idx = meshes.faces_packed_to_mesh_idx() # (sum(T_n),) + num_tris_per_mesh = meshes.num_faces_per_mesh() # (N, ) + weights_t = num_tris_per_mesh.gather(0, tri_to_mesh_idx) + weights_t = 1.0 / weights_t.float() + face_to_point = face_to_point * weights_t + face_dist = face_to_point.sum() / N + + return point_dist + face_dist diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/__init__.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..95f5c65aeba630bfa51ffb4575cf32003c8fb7f2 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/__init__.py @@ -0,0 +1,49 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from .ball_query import ball_query +from .cameras_alignment import corresponding_cameras_alignment + +from .cubify import cubify +from .graph_conv import GraphConv +from .interp_face_attrs import interpolate_face_attributes +from .iou_box3d import box3d_overlap +from .knn import knn_gather, knn_points +from .laplacian_matrices import cot_laplacian, laplacian, norm_laplacian + +from .mesh_face_areas_normals import mesh_face_areas_normals +from .mesh_filtering import taubin_smoothing + +from .packed_to_padded import packed_to_padded, padded_to_packed +from .perspective_n_points import efficient_pnp +from .points_alignment import corresponding_points_alignment, iterative_closest_point +from .points_normals import ( + estimate_pointcloud_local_coord_frames, + estimate_pointcloud_normals, +) +from .points_to_volumes import ( + add_pointclouds_to_volumes, + add_points_features_to_volume_densities_features, +) + +from .sample_farthest_points import sample_farthest_points + +from .sample_points_from_meshes import sample_points_from_meshes +from .subdivide_meshes import SubdivideMeshes +from .utils import ( + convert_pointclouds_to_tensor, + eyes, + get_point_covariances, + is_pointclouds, + wmean, +) + +from .vert_align import vert_align + + +__all__ = [k for k in globals().keys() if not k.startswith("_")] diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/ball_query.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/ball_query.py new file mode 100644 index 0000000000000000000000000000000000000000..31266c4d2b6ad36f27086ff5f108ba6b464fe698 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/ball_query.py @@ -0,0 +1,142 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from typing import Union + +import torch +from pytorch3d import _C +from torch.autograd import Function +from torch.autograd.function import once_differentiable + +from .knn import _KNN +from .utils import masked_gather + + +class _ball_query(Function): + """ + Torch autograd Function wrapper for Ball Query C++/CUDA implementations. + """ + + @staticmethod + def forward(ctx, p1, p2, lengths1, lengths2, K, radius): + """ + Arguments defintions the same as in the ball_query function + """ + idx, dists = _C.ball_query(p1, p2, lengths1, lengths2, K, radius) + ctx.save_for_backward(p1, p2, lengths1, lengths2, idx) + ctx.mark_non_differentiable(idx) + return dists, idx + + @staticmethod + @once_differentiable + def backward(ctx, grad_dists, grad_idx): + p1, p2, lengths1, lengths2, idx = ctx.saved_tensors + # TODO(gkioxari) Change cast to floats once we add support for doubles. + if not (grad_dists.dtype == torch.float32): + grad_dists = grad_dists.float() + if not (p1.dtype == torch.float32): + p1 = p1.float() + if not (p2.dtype == torch.float32): + p2 = p2.float() + + # Reuse the KNN backward function + # by default, norm is 2 + grad_p1, grad_p2 = _C.knn_points_backward( + p1, p2, lengths1, lengths2, idx, 2, grad_dists + ) + return grad_p1, grad_p2, None, None, None, None + + +def ball_query( + p1: torch.Tensor, + p2: torch.Tensor, + lengths1: Union[torch.Tensor, None] = None, + lengths2: Union[torch.Tensor, None] = None, + K: int = 500, + radius: float = 0.2, + return_nn: bool = True, +): + """ + Ball Query is an alternative to KNN. It can be + used to find all points in p2 that are within a specified radius + to the query point in p1 (with an upper limit of K neighbors). + + The neighbors returned are not necssarily the *nearest* to the + point in p1, just the first K values in p2 which are within the + specified radius. + + This method is faster than kNN when there are large numbers of points + in p2 and the ordering of neighbors is not important compared to the + distance being within the radius threshold. + + "Ball query’s local neighborhood guarantees a fixed region scale thus + making local region features more generalizable across space, which is + preferred for tasks requiring local pattern recognition + (e.g. semantic point labeling)" [1]. + + [1] Charles R. Qi et al, "PointNet++: Deep Hierarchical Feature Learning + on Point Sets in a Metric Space", NeurIPS 2017. + + Args: + p1: Tensor of shape (N, P1, D) giving a batch of N point clouds, each + containing up to P1 points of dimension D. These represent the centers of + the ball queries. + p2: Tensor of shape (N, P2, D) giving a batch of N point clouds, each + containing up to P2 points of dimension D. + lengths1: LongTensor of shape (N,) of values in the range [0, P1], giving the + length of each pointcloud in p1. Or None to indicate that every cloud has + length P1. + lengths2: LongTensor of shape (N,) of values in the range [0, P2], giving the + length of each pointcloud in p2. Or None to indicate that every cloud has + length P2. + K: Integer giving the upper bound on the number of samples to take + within the radius + radius: the radius around each point within which the neighbors need to be located + return_nn: If set to True returns the K neighbor points in p2 for each point in p1. + + Returns: + dists: Tensor of shape (N, P1, K) giving the squared distances to + the neighbors. This is padded with zeros both where a cloud in p2 + has fewer than S points and where a cloud in p1 has fewer than P1 points + and also if there are fewer than K points which satisfy the radius threshold. + + idx: LongTensor of shape (N, P1, K) giving the indices of the + S neighbors in p2 for points in p1. + Concretely, if `p1_idx[n, i, k] = j` then `p2[n, j]` is the k-th + neighbor to `p1[n, i]` in `p2[n]`. This is padded with -1 both where a cloud + in p2 has fewer than S points and where a cloud in p1 has fewer than P1 + points and also if there are fewer than K points which satisfy the radius threshold. + + nn: Tensor of shape (N, P1, K, D) giving the K neighbors in p2 for + each point in p1. Concretely, `p2_nn[n, i, k]` gives the k-th neighbor + for `p1[n, i]`. Returned if `return_nn` is True. The output is a tensor + of shape (N, P1, K, U). + + """ + if p1.shape[0] != p2.shape[0]: + raise ValueError("pts1 and pts2 must have the same batch dimension.") + if p1.shape[2] != p2.shape[2]: + raise ValueError("pts1 and pts2 must have the same point dimension.") + + p1 = p1.contiguous() + p2 = p2.contiguous() + P1 = p1.shape[1] + P2 = p2.shape[1] + N = p1.shape[0] + + if lengths1 is None: + lengths1 = torch.full((N,), P1, dtype=torch.int64, device=p1.device) + if lengths2 is None: + lengths2 = torch.full((N,), P2, dtype=torch.int64, device=p1.device) + + dists, idx = _ball_query.apply(p1, p2, lengths1, lengths2, K, radius) + + # Gather the neighbors if needed + points_nn = masked_gather(p2, idx) if return_nn else None + + return _KNN(dists=dists, idx=idx, knn=points_nn) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/cameras_alignment.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/cameras_alignment.py new file mode 100644 index 0000000000000000000000000000000000000000..40e00aba10d2281df3d59d9f1185b12142f03d3e --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/cameras_alignment.py @@ -0,0 +1,224 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from typing import TYPE_CHECKING + +import torch + +from .. import ops + + +if TYPE_CHECKING: + from pytorch3d.renderer.cameras import CamerasBase + + +def corresponding_cameras_alignment( + cameras_src: "CamerasBase", + cameras_tgt: "CamerasBase", + estimate_scale: bool = True, + mode: str = "extrinsics", + eps: float = 1e-9, +) -> "CamerasBase": # pragma: no cover + """ + .. warning:: + The `corresponding_cameras_alignment` API is experimental + and subject to change! + + Estimates a single similarity transformation between two sets of cameras + `cameras_src` and `cameras_tgt` and returns an aligned version of + `cameras_src`. + + Given source cameras [(R_1, T_1), (R_2, T_2), ..., (R_N, T_N)] and + target cameras [(R_1', T_1'), (R_2', T_2'), ..., (R_N', T_N')], + where (R_i, T_i) is a 2-tuple of the camera rotation and translation matrix + respectively, the algorithm finds a global rotation, translation and scale + (R_A, T_A, s_A) which aligns all source cameras with the target cameras + such that the following holds: + + Under the change of coordinates using a similarity transform + (R_A, T_A, s_A) a 3D point X' is mapped to X with: :: + + X = (X' R_A + T_A) / s_A + + Then, for all cameras `i`, we assume that the following holds: :: + + X R_i + T_i = s' (X' R_i' + T_i'), + + i.e. an adjusted point X' is mapped by a camera (R_i', T_i') + to the same point as imaged from camera (R_i, T_i) after resolving + the scale ambiguity with a global scalar factor s'. + + Substituting for X above gives rise to the following: :: + + (X' R_A + T_A) / s_A R_i + T_i = s' (X' R_i' + T_i') // · s_A + (X' R_A + T_A) R_i + T_i s_A = (s' s_A) (X' R_i' + T_i') + s' := 1 / s_A # without loss of generality + (X' R_A + T_A) R_i + T_i s_A = X' R_i' + T_i' + X' R_A R_i + T_A R_i + T_i s_A = X' R_i' + T_i' + ^^^^^^^ ^^^^^^^^^^^^^^^^^ + ~= R_i' ~= T_i' + + i.e. after estimating R_A, T_A, s_A, the aligned source cameras have + extrinsics: :: + + cameras_src_align = (R_A R_i, T_A R_i + T_i s_A) ~= (R_i', T_i') + + We support two ways `R_A, T_A, s_A` can be estimated: + 1) `mode=='centers'` + Estimates the similarity alignment between camera centers using + Umeyama's algorithm (see `pytorch3d.ops.corresponding_points_alignment` + for details) and transforms camera extrinsics accordingly. + + 2) `mode=='extrinsics'` + Defines the alignment problem as a system + of the following equations: :: + + for all i: + [ R_A 0 ] x [ R_i 0 ] = [ R_i' 0 ] + [ T_A^T 1 ] [ (s_A T_i^T) 1 ] [ T_i' 1 ] + + `R_A, T_A` and `s_A` are then obtained by solving the + system in the least squares sense. + + The estimated camera transformation is a true similarity transform, i.e. + it cannot be a reflection. + + Args: + cameras_src: `N` cameras to be aligned. + cameras_tgt: `N` target cameras. + estimate_scale: Controls whether the alignment transform is rigid + (`estimate_scale=False`), or a similarity (`estimate_scale=True`). + `s_A` is set to `1` if `estimate_scale==False`. + mode: Controls the alignment algorithm. + Can be one either `'centers'` or `'extrinsics'`. Please refer to the + description above for details. + eps: A scalar for clamping to avoid dividing by zero. + Active when `estimate_scale==True`. + + Returns: + cameras_src_aligned: `cameras_src` after applying the alignment transform. + """ + + if cameras_src.R.shape[0] != cameras_tgt.R.shape[0]: + raise ValueError( + "cameras_src and cameras_tgt have to contain the same number of cameras!" + ) + + if mode == "centers": + align_fun = _align_camera_centers + elif mode == "extrinsics": + align_fun = _align_camera_extrinsics + else: + raise ValueError("mode has to be one of (centers, extrinsics)") + + align_t_R, align_t_T, align_t_s = align_fun( + cameras_src, cameras_tgt, estimate_scale=estimate_scale, eps=eps + ) + + # create a new cameras object and set the R and T accordingly + cameras_src_aligned = cameras_src.clone() + cameras_src_aligned.R = torch.bmm(align_t_R.expand_as(cameras_src.R), cameras_src.R) + cameras_src_aligned.T = ( + torch.bmm( + align_t_T[:, None].repeat(cameras_src.R.shape[0], 1, 1), + cameras_src.R, + )[:, 0] + + cameras_src.T * align_t_s + ) + + return cameras_src_aligned + + +def _align_camera_centers( + cameras_src: "CamerasBase", + cameras_tgt: "CamerasBase", + estimate_scale: bool = True, + eps: float = 1e-9, +): # pragma: no cover + """ + Use Umeyama's algorithm to align the camera centers. + """ + centers_src = cameras_src.get_camera_center() + centers_tgt = cameras_tgt.get_camera_center() + align_t = ops.corresponding_points_alignment( + centers_src[None], + centers_tgt[None], + estimate_scale=estimate_scale, + allow_reflection=False, + eps=eps, + ) + # the camera transform is the inverse of the estimated transform between centers + align_t_R = align_t.R.permute(0, 2, 1) + align_t_T = -(torch.bmm(align_t.T[:, None], align_t_R))[:, 0] + align_t_s = align_t.s[0] + + return align_t_R, align_t_T, align_t_s + + +def _align_camera_extrinsics( + cameras_src: "CamerasBase", + cameras_tgt: "CamerasBase", + estimate_scale: bool = True, + eps: float = 1e-9, +): # pragma: no cover + """ + Get the global rotation R_A with svd of cov(RR^T): + ``` + R_A R_i = R_i' for all i + R_A [R_1 R_2 ... R_N] = [R_1' R_2' ... R_N'] + U, _, V = svd([R_1 R_2 ... R_N]^T [R_1' R_2' ... R_N']) + R_A = (U V^T)^T + ``` + """ + RRcov = torch.bmm(cameras_src.R, cameras_tgt.R.transpose(2, 1)).mean(0) + U, _, V = torch.svd(RRcov) + align_t_R = V @ U.t() + + """ + The translation + scale `T_A` and `s_A` is computed by finding + a translation and scaling that aligns two tensors `A, B` + defined as follows: + ``` + T_A R_i + s_A T_i = T_i' ; for all i // · R_i^T + s_A T_i R_i^T + T_A = T_i' R_i^T ; for all i + ^^^^^^^^^ ^^^^^^^^^^ + A_i B_i + + A_i := T_i R_i^T + A = [A_1 A_2 ... A_N] + B_i := T_i' R_i^T + B = [B_1 B_2 ... B_N] + ``` + The scale s_A can be retrieved by matching the correlations of + the points sets A and B: + ``` + s_A = (A-mean(A))*(B-mean(B)).sum() / ((A-mean(A))**2).sum() + ``` + The translation `T_A` is then defined as: + ``` + T_A = mean(B) - mean(A) * s_A + ``` + """ + A = torch.bmm(cameras_src.R, cameras_src.T[:, :, None])[:, :, 0] + B = torch.bmm(cameras_src.R, cameras_tgt.T[:, :, None])[:, :, 0] + Amu = A.mean(0, keepdim=True) + Bmu = B.mean(0, keepdim=True) + if estimate_scale and A.shape[0] > 1: + # get the scaling component by matching covariances + # of centered A and centered B + Ac = A - Amu + Bc = B - Bmu + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and `int`. + align_t_s = (Ac * Bc).mean() / (Ac**2).mean().clamp(eps) + else: + # set the scale to identity + align_t_s = 1.0 + # get the translation as the difference between the means of A and B + align_t_T = Bmu - align_t_s * Amu + + return align_t_R, align_t_T, align_t_s diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/cubify.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/cubify.py new file mode 100644 index 0000000000000000000000000000000000000000..ceab4bc70de23fc42364d5a670be8b92e6ea5dfc --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/cubify.py @@ -0,0 +1,275 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +from typing import Optional + +import torch +import torch.nn.functional as F + +from pytorch3d.common.compat import meshgrid_ij + +from pytorch3d.structures import Meshes + + +def unravel_index(idx, dims) -> torch.Tensor: + r""" + Equivalent to np.unravel_index + Args: + idx: A LongTensor whose elements are indices into the + flattened version of an array of dimensions dims. + dims: The shape of the array to be indexed. + Implemented only for dims=(N, H, W, D) + """ + if len(dims) != 4: + raise ValueError("Expects a 4-element list.") + N, H, W, D = dims + n = idx // (H * W * D) + h = (idx - n * H * W * D) // (W * D) + w = (idx - n * H * W * D - h * W * D) // D + d = idx - n * H * W * D - h * W * D - w * D + return torch.stack((n, h, w, d), dim=1) + + +def ravel_index(idx, dims) -> torch.Tensor: + """ + Computes the linear index in an array of shape dims. + It performs the reverse functionality of unravel_index + Args: + idx: A LongTensor of shape (N, 3). Each row corresponds to indices into an + array of dimensions dims. + dims: The shape of the array to be indexed. + Implemented only for dims=(H, W, D) + """ + if len(dims) != 3: + raise ValueError("Expects a 3-element list") + if idx.shape[1] != 3: + raise ValueError("Expects an index tensor of shape Nx3") + H, W, D = dims + linind = idx[:, 0] * W * D + idx[:, 1] * D + idx[:, 2] + return linind + + +@torch.no_grad() +def cubify( + voxels: torch.Tensor, + thresh: float, + *, + feats: Optional[torch.Tensor] = None, + device=None, + align: str = "topleft" +) -> Meshes: + r""" + Converts a voxel to a mesh by replacing each occupied voxel with a cube + consisting of 12 faces and 8 vertices. Shared vertices are merged, and + internal faces are removed. + Args: + voxels: A FloatTensor of shape (N, D, H, W) containing occupancy probabilities. + thresh: A scalar threshold. If a voxel occupancy is larger than + thresh, the voxel is considered occupied. + feats: A FloatTensor of shape (N, K, D, H, W) containing the color information + of each voxel. K is the number of channels. This is supported only when + align == "center" + device: The device of the output meshes + align: Defines the alignment of the mesh vertices and the grid locations. + Has to be one of {"topleft", "corner", "center"}. See below for explanation. + Default is "topleft". + Returns: + meshes: A Meshes object of the corresponding meshes. + + + The alignment between the vertices of the cubified mesh and the voxel locations (or pixels) + is defined by the choice of `align`. We support three modes, as shown below for a 2x2 grid: + + X---X---- X-------X --------- + | | | | | | | X | X | + X---X---- --------- --------- + | | | | | | | X | X | + --------- X-------X --------- + + topleft corner center + + In the figure, X denote the grid locations and the squares represent the added cuboids. + When `align="topleft"`, then the top left corner of each cuboid corresponds to the + pixel coordinate of the input grid. + When `align="corner"`, then the corners of the output mesh span the whole grid. + When `align="center"`, then the grid locations form the center of the cuboids. + """ + + if device is None: + device = voxels.device + + if align not in ["topleft", "corner", "center"]: + raise ValueError("Align mode must be one of (topleft, corner, center).") + + if len(voxels) == 0: + return Meshes(verts=[], faces=[]) + + N, D, H, W = voxels.size() + # vertices corresponding to a unit cube: 8x3 + cube_verts = torch.tensor( + [ + [0, 0, 0], + [0, 0, 1], + [0, 1, 0], + [0, 1, 1], + [1, 0, 0], + [1, 0, 1], + [1, 1, 0], + [1, 1, 1], + ], + dtype=torch.int64, + device=device, + ) + + # faces corresponding to a unit cube: 12x3 + cube_faces = torch.tensor( + [ + [0, 1, 2], + [1, 3, 2], # left face: 0, 1 + [2, 3, 6], + [3, 7, 6], # bottom face: 2, 3 + [0, 2, 6], + [0, 6, 4], # front face: 4, 5 + [0, 5, 1], + [0, 4, 5], # up face: 6, 7 + [6, 7, 5], + [6, 5, 4], # right face: 8, 9 + [1, 7, 3], + [1, 5, 7], # back face: 10, 11 + ], + dtype=torch.int64, + device=device, + ) + + wx = torch.tensor([0.5, 0.5], device=device).view(1, 1, 1, 1, 2) + wy = torch.tensor([0.5, 0.5], device=device).view(1, 1, 1, 2, 1) + wz = torch.tensor([0.5, 0.5], device=device).view(1, 1, 2, 1, 1) + + voxelt = voxels.ge(thresh).float() + # N x 1 x D x H x W + voxelt = voxelt.view(N, 1, D, H, W) + + # N x 1 x (D-1) x (H-1) x (W-1) + voxelt_x = F.conv3d(voxelt, wx).gt(0.5).float() + voxelt_y = F.conv3d(voxelt, wy).gt(0.5).float() + voxelt_z = F.conv3d(voxelt, wz).gt(0.5).float() + + # 12 x N x 1 x D x H x W + faces_idx = torch.ones((cube_faces.size(0), N, 1, D, H, W), device=device) + + # add left face + faces_idx[0, :, :, :, :, 1:] = 1 - voxelt_x + faces_idx[1, :, :, :, :, 1:] = 1 - voxelt_x + # add bottom face + faces_idx[2, :, :, :, :-1, :] = 1 - voxelt_y + faces_idx[3, :, :, :, :-1, :] = 1 - voxelt_y + # add front face + faces_idx[4, :, :, 1:, :, :] = 1 - voxelt_z + faces_idx[5, :, :, 1:, :, :] = 1 - voxelt_z + # add up face + faces_idx[6, :, :, :, 1:, :] = 1 - voxelt_y + faces_idx[7, :, :, :, 1:, :] = 1 - voxelt_y + # add right face + faces_idx[8, :, :, :, :, :-1] = 1 - voxelt_x + faces_idx[9, :, :, :, :, :-1] = 1 - voxelt_x + # add back face + faces_idx[10, :, :, :-1, :, :] = 1 - voxelt_z + faces_idx[11, :, :, :-1, :, :] = 1 - voxelt_z + + faces_idx *= voxelt + + # N x H x W x D x 12 + faces_idx = faces_idx.permute(1, 2, 4, 5, 3, 0).squeeze(1) + # (NHWD) x 12 + faces_idx = faces_idx.contiguous() + faces_idx = faces_idx.view(-1, cube_faces.size(0)) + + # boolean to linear index + # NF x 2 + linind = torch.nonzero(faces_idx, as_tuple=False) + + # NF x 4 + nyxz = unravel_index(linind[:, 0], (N, H, W, D)) + + # NF x 3: faces + faces = torch.index_select(cube_faces, 0, linind[:, 1]) + + grid_faces = [] + for d in range(cube_faces.size(1)): + # NF x 3 + xyz = torch.index_select(cube_verts, 0, faces[:, d]) + permute_idx = torch.tensor([1, 0, 2], device=device) + yxz = torch.index_select(xyz, 1, permute_idx) + yxz += nyxz[:, 1:] + # NF x 1 + temp = ravel_index(yxz, (H + 1, W + 1, D + 1)) + grid_faces.append(temp) + # NF x 3 + grid_faces = torch.stack(grid_faces, dim=1) + + y, x, z = meshgrid_ij(torch.arange(H + 1), torch.arange(W + 1), torch.arange(D + 1)) + y = y.to(device=device, dtype=torch.float32) + x = x.to(device=device, dtype=torch.float32) + z = z.to(device=device, dtype=torch.float32) + + if align == "center": + x = x - 0.5 + y = y - 0.5 + z = z - 0.5 + + margin = 0.0 if align == "corner" else 1.0 + y = y * 2.0 / (H - margin) - 1.0 + x = x * 2.0 / (W - margin) - 1.0 + z = z * 2.0 / (D - margin) - 1.0 + + # ((H+1)(W+1)(D+1)) x 3 + grid_verts = torch.stack((x, y, z), dim=3).view(-1, 3) + + if len(nyxz) == 0: + verts_list = [torch.tensor([], dtype=torch.float32, device=device)] * N + faces_list = [torch.tensor([], dtype=torch.int64, device=device)] * N + return Meshes(verts=verts_list, faces=faces_list) + + num_verts = grid_verts.size(0) + grid_faces += nyxz[:, 0].view(-1, 1) * num_verts + idleverts = torch.ones(num_verts * N, dtype=torch.uint8, device=device) + + indices = grid_faces.flatten() + if device.type == "cpu": + indices = torch.unique(indices) + idleverts.scatter_(0, indices, 0) + grid_faces -= nyxz[:, 0].view(-1, 1) * num_verts + split_size = torch.bincount(nyxz[:, 0], minlength=N) + faces_list = list(torch.split(grid_faces, split_size.tolist(), 0)) + + idleverts = idleverts.view(N, num_verts) + idlenum = idleverts.cumsum(1) + + verts_list = [ + grid_verts.index_select(0, (idleverts[n] == 0).nonzero(as_tuple=False)[:, 0]) + for n in range(N) + ] + + textures_list = None + if feats is not None and align == "center": + # We return a TexturesAtlas containing one color for each face + # N x K x D x H x W -> N x H x W x D x K + feats = feats.permute(0, 3, 4, 2, 1) + + # (NHWD) x K + feats = feats.reshape(-1, feats.size(4)) + feats = torch.index_select(feats, 0, linind[:, 0]) + feats = feats.reshape(-1, 1, 1, feats.size(1)) + feats_list = list(torch.split(feats, split_size.tolist(), 0)) + from pytorch3d.renderer.mesh.textures import TexturesAtlas + + textures_list = TexturesAtlas(feats_list) + + faces_list = [nface - idlenum[n][nface] for n, nface in enumerate(faces_list)] + return Meshes(verts=verts_list, faces=faces_list, textures=textures_list) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/graph_conv.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/graph_conv.py new file mode 100644 index 0000000000000000000000000000000000000000..a43eeb3ad204ff0dde46b8945383c15367e0d788 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/graph_conv.py @@ -0,0 +1,176 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +import torch +import torch.nn as nn +from pytorch3d import _C +from torch.autograd import Function +from torch.autograd.function import once_differentiable + + +class GraphConv(nn.Module): + """A single graph convolution layer.""" + + def __init__( + self, + input_dim: int, + output_dim: int, + init: str = "normal", + directed: bool = False, + ) -> None: + """ + Args: + input_dim: Number of input features per vertex. + output_dim: Number of output features per vertex. + init: Weight initialization method. Can be one of ['zero', 'normal']. + directed: Bool indicating if edges in the graph are directed. + """ + super().__init__() + self.input_dim = input_dim + self.output_dim = output_dim + self.directed = directed + self.w0 = nn.Linear(input_dim, output_dim) + self.w1 = nn.Linear(input_dim, output_dim) + + if init == "normal": + nn.init.normal_(self.w0.weight, mean=0, std=0.01) + nn.init.normal_(self.w1.weight, mean=0, std=0.01) + self.w0.bias.data.zero_() + self.w1.bias.data.zero_() + elif init == "zero": + self.w0.weight.data.zero_() + self.w1.weight.data.zero_() + else: + raise ValueError('Invalid GraphConv initialization "%s"' % init) + + def forward(self, verts, edges): + """ + Args: + verts: FloatTensor of shape (V, input_dim) where V is the number of + vertices and input_dim is the number of input features + per vertex. input_dim has to match the input_dim specified + in __init__. + edges: LongTensor of shape (E, 2) where E is the number of edges + where each edge has the indices of the two vertices which + form the edge. + + Returns: + out: FloatTensor of shape (V, output_dim) where output_dim is the + number of output features per vertex. + """ + if verts.is_cuda != edges.is_cuda: + raise ValueError("verts and edges tensors must be on the same device.") + if verts.shape[0] == 0: + # empty graph. + return verts.new_zeros((0, self.output_dim)) * verts.sum() + + verts_w0 = self.w0(verts) # (V, output_dim) + verts_w1 = self.w1(verts) # (V, output_dim) + + if torch.cuda.is_available() and verts.is_cuda and edges.is_cuda: + neighbor_sums = gather_scatter(verts_w1, edges, self.directed) + else: + neighbor_sums = gather_scatter_python( + verts_w1, edges, self.directed + ) # (V, output_dim) + + # Add neighbor features to each vertex's features. + out = verts_w0 + neighbor_sums + return out + + def __repr__(self): + Din, Dout, directed = self.input_dim, self.output_dim, self.directed + return "GraphConv(%d -> %d, directed=%r)" % (Din, Dout, directed) + + +def gather_scatter_python(input, edges, directed: bool = False): + """ + Python implementation of gather_scatter for aggregating features of + neighbor nodes in a graph. + + Given a directed graph: v0 -> v1 -> v2 the updated feature for v1 depends + on v2 in order to be consistent with Morris et al. AAAI 2019 + (https://arxiv.org/abs/1810.02244). This only affects + directed graphs; for undirected graphs v1 will depend on both v0 and v2, + no matter which way the edges are physically stored. + + Args: + input: Tensor of shape (num_vertices, input_dim). + edges: Tensor of edge indices of shape (num_edges, 2). + directed: bool indicating if edges are directed. + + Returns: + output: Tensor of same shape as input. + """ + if not (input.dim() == 2): + raise ValueError("input can only have 2 dimensions.") + if not (edges.dim() == 2): + raise ValueError("edges can only have 2 dimensions.") + if not (edges.shape[1] == 2): + raise ValueError("edges must be of shape (num_edges, 2).") + + num_vertices, input_feature_dim = input.shape + num_edges = edges.shape[0] + output = torch.zeros_like(input) + idx0 = edges[:, 0].view(num_edges, 1).expand(num_edges, input_feature_dim) + idx1 = edges[:, 1].view(num_edges, 1).expand(num_edges, input_feature_dim) + + output = output.scatter_add(0, idx0, input.gather(0, idx1)) + if not directed: + output = output.scatter_add(0, idx1, input.gather(0, idx0)) + return output + + +class GatherScatter(Function): + """ + Torch autograd Function wrapper for gather_scatter C++/CUDA implementations. + """ + + @staticmethod + def forward(ctx, input, edges, directed=False): + """ + Args: + ctx: Context object used to calculate gradients. + input: Tensor of shape (num_vertices, input_dim) + edges: Tensor of edge indices of shape (num_edges, 2) + directed: Bool indicating if edges are directed. + + Returns: + output: Tensor of same shape as input. + """ + if not (input.dim() == 2): + raise ValueError("input can only have 2 dimensions.") + if not (edges.dim() == 2): + raise ValueError("edges can only have 2 dimensions.") + if not (edges.shape[1] == 2): + raise ValueError("edges must be of shape (num_edges, 2).") + if not (input.dtype == torch.float32): + raise ValueError("input has to be of type torch.float32.") + + ctx.directed = directed + input, edges = input.contiguous(), edges.contiguous() + ctx.save_for_backward(edges) + backward = False + output = _C.gather_scatter(input, edges, directed, backward) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + grad_output = grad_output.contiguous() + edges = ctx.saved_tensors[0] + directed = ctx.directed + backward = True + grad_input = _C.gather_scatter(grad_output, edges, directed, backward) + grad_edges = None + grad_directed = None + return grad_input, grad_edges, grad_directed + + +gather_scatter = GatherScatter.apply diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/interp_face_attrs.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/interp_face_attrs.py new file mode 100644 index 0000000000000000000000000000000000000000..c1f04891233e2566e93066f0057f2da4afdbb7cf --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/interp_face_attrs.py @@ -0,0 +1,101 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import torch +from pytorch3d import _C +from torch.autograd import Function +from torch.autograd.function import once_differentiable + + +def interpolate_face_attributes( + pix_to_face: torch.Tensor, + barycentric_coords: torch.Tensor, + face_attributes: torch.Tensor, +) -> torch.Tensor: + """ + Interpolate arbitrary face attributes using the barycentric coordinates + for each pixel in the rasterized output. + + Args: + pix_to_face: LongTensor of shape (...) specifying the indices + of the faces (in the packed representation) which overlap each + pixel in the image. A value < 0 indicates that the pixel does not + overlap any face and should be skipped. + barycentric_coords: FloatTensor of shape (N, H, W, K, 3) specifying + the barycentric coordinates of each pixel + relative to the faces (in the packed + representation) which overlap the pixel. + face_attributes: packed attributes of shape (total_faces, 3, D), + specifying the value of the attribute for each + vertex in the face. + + Returns: + pixel_vals: tensor of shape (N, H, W, K, D) giving the interpolated + value of the face attribute for each pixel. + """ + # Check shapes + F, FV, D = face_attributes.shape + if FV != 3: + raise ValueError("Faces can only have three vertices; got %r" % FV) + N, H, W, K, _ = barycentric_coords.shape + if pix_to_face.shape != (N, H, W, K): + msg = "pix_to_face must have shape (batch_size, H, W, K); got %r" + raise ValueError(msg % (pix_to_face.shape,)) + + # On CPU use the python version + # TODO: Implement a C++ version of this function + if not pix_to_face.is_cuda: + args = (pix_to_face, barycentric_coords, face_attributes) + return interpolate_face_attributes_python(*args) + + # Otherwise flatten and call the custom autograd function + N, H, W, K = pix_to_face.shape + pix_to_face = pix_to_face.view(-1) + barycentric_coords = barycentric_coords.view(N * H * W * K, 3) + args = (pix_to_face, barycentric_coords, face_attributes) + out = _InterpFaceAttrs.apply(*args) + out = out.view(N, H, W, K, -1) + return out + + +class _InterpFaceAttrs(Function): + @staticmethod + def forward(ctx, pix_to_face, barycentric_coords, face_attrs): + args = (pix_to_face, barycentric_coords, face_attrs) + ctx.save_for_backward(*args) + return _C.interp_face_attrs_forward(*args) + + @staticmethod + @once_differentiable + def backward(ctx, grad_pix_attrs): + args = ctx.saved_tensors + args = args + (grad_pix_attrs,) + grads = _C.interp_face_attrs_backward(*args) + grad_pix_to_face = None + grad_barycentric_coords = grads[0] + grad_face_attrs = grads[1] + return grad_pix_to_face, grad_barycentric_coords, grad_face_attrs + + +def interpolate_face_attributes_python( + pix_to_face: torch.Tensor, + barycentric_coords: torch.Tensor, + face_attributes: torch.Tensor, +) -> torch.Tensor: + F, FV, D = face_attributes.shape + N, H, W, K, _ = barycentric_coords.shape + + # Replace empty pixels in pix_to_face with 0 in order to interpolate. + mask = pix_to_face < 0 + pix_to_face = pix_to_face.clone() + pix_to_face[mask] = 0 + idx = pix_to_face.view(N * H * W * K, 1, 1).expand(N * H * W * K, 3, D) + pixel_face_vals = face_attributes.gather(0, idx).view(N, H, W, K, 3, D) + pixel_vals = (barycentric_coords[..., None] * pixel_face_vals).sum(dim=-2) + pixel_vals[mask] = 0 # Replace masked values in output. + return pixel_vals diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/iou_box3d.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/iou_box3d.py new file mode 100644 index 0000000000000000000000000000000000000000..3c01bd65f8fc4f28d3b55bb5ba993c10a6c5cd7c --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/iou_box3d.py @@ -0,0 +1,168 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from typing import Tuple + +import torch +import torch.nn.functional as F +from pytorch3d import _C +from torch.autograd import Function + + +# -------------------------------------------------- # +# CONSTANTS # +# -------------------------------------------------- # +""" +_box_planes and _box_triangles define the 4- and 3-connectivity +of the 8 box corners. +_box_planes gives the quad faces of the 3D box +_box_triangles gives the triangle faces of the 3D box +""" +_box_planes = [ + [0, 1, 2, 3], + [3, 2, 6, 7], + [0, 1, 5, 4], + [0, 3, 7, 4], + [1, 2, 6, 5], + [4, 5, 6, 7], +] +_box_triangles = [ + [0, 1, 2], + [0, 3, 2], + [4, 5, 6], + [4, 6, 7], + [1, 5, 6], + [1, 6, 2], + [0, 4, 7], + [0, 7, 3], + [3, 2, 6], + [3, 6, 7], + [0, 1, 5], + [0, 4, 5], +] + + +def _check_coplanar(boxes: torch.Tensor, eps: float = 1e-4) -> None: + faces = torch.tensor(_box_planes, dtype=torch.int64, device=boxes.device) + verts = boxes.index_select(index=faces.view(-1), dim=1) + B = boxes.shape[0] + P, V = faces.shape + # (B, P, 4, 3) -> (B, P, 3) + v0, v1, v2, v3 = verts.reshape(B, P, V, 3).unbind(2) + + # Compute the normal + e0 = F.normalize(v1 - v0, dim=-1) + e1 = F.normalize(v2 - v0, dim=-1) + normal = F.normalize(torch.cross(e0, e1, dim=-1), dim=-1) + + # Check the fourth vertex is also on the same plane + mat1 = (v3 - v0).view(B, 1, -1) # (B, 1, P*3) + mat2 = normal.view(B, -1, 1) # (B, P*3, 1) + if not (mat1.bmm(mat2).abs() < eps).all().item(): + msg = "Plane vertices are not coplanar" + raise ValueError(msg) + + return + + +def _check_nonzero(boxes: torch.Tensor, eps: float = 1e-4) -> None: + """ + Checks that the sides of the box have a non zero area + """ + faces = torch.tensor(_box_triangles, dtype=torch.int64, device=boxes.device) + verts = boxes.index_select(index=faces.view(-1), dim=1) + B = boxes.shape[0] + T, V = faces.shape + # (B, T, 3, 3) -> (B, T, 3) + v0, v1, v2 = verts.reshape(B, T, V, 3).unbind(2) + + normals = torch.cross(v1 - v0, v2 - v0, dim=-1) # (B, T, 3) + face_areas = normals.norm(dim=-1) / 2 + + if (face_areas < eps).any().item(): + msg = "Planes have zero areas" + raise ValueError(msg) + + return + + +class _box3d_overlap(Function): + """ + Torch autograd Function wrapper for box3d_overlap C++/CUDA implementations. + Backward is not supported. + """ + + @staticmethod + def forward(ctx, boxes1, boxes2): + """ + Arguments defintions the same as in the box3d_overlap function + """ + vol, iou = _C.iou_box3d(boxes1, boxes2) + return vol, iou + + @staticmethod + def backward(ctx, grad_vol, grad_iou): + raise ValueError("box3d_overlap backward is not supported") + + +def box3d_overlap( + boxes1: torch.Tensor, boxes2: torch.Tensor, eps: float = 1e-4 +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Computes the intersection of 3D boxes1 and boxes2. + + Inputs boxes1, boxes2 are tensors of shape (B, 8, 3) + (where B doesn't have to be the same for boxes1 and boxes2), + containing the 8 corners of the boxes, as follows: + + (4) +---------+. (5) + | ` . | ` . + | (0) +---+-----+ (1) + | | | | + (7) +-----+---+. (6)| + ` . | ` . | + (3) ` +---------+ (2) + + + NOTE: Throughout this implementation, we assume that boxes + are defined by their 8 corners exactly in the order specified in the + diagram above for the function to give correct results. In addition + the vertices on each plane must be coplanar. + As an alternative to the diagram, this is a unit bounding + box which has the correct vertex ordering: + + box_corner_vertices = [ + [0, 0, 0], + [1, 0, 0], + [1, 1, 0], + [0, 1, 0], + [0, 0, 1], + [1, 0, 1], + [1, 1, 1], + [0, 1, 1], + ] + + Args: + boxes1: tensor of shape (N, 8, 3) of the coordinates of the 1st boxes + boxes2: tensor of shape (M, 8, 3) of the coordinates of the 2nd boxes + Returns: + vol: (N, M) tensor of the volume of the intersecting convex shapes + iou: (N, M) tensor of the intersection over union which is + defined as: `iou = vol / (vol1 + vol2 - vol)` + """ + if not all((8, 3) == box.shape[1:] for box in [boxes1, boxes2]): + raise ValueError("Each box in the batch must be of shape (8, 3)") + + _check_coplanar(boxes1, eps) + _check_coplanar(boxes2, eps) + _check_nonzero(boxes1, eps) + _check_nonzero(boxes2, eps) + + vol, iou = _box3d_overlap.apply(boxes1, boxes2) + + return vol, iou diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/knn.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/knn.py new file mode 100644 index 0000000000000000000000000000000000000000..114334fdab5832562327b47ac8d1c34271961ff4 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/knn.py @@ -0,0 +1,250 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from collections import namedtuple +from typing import Union + +import torch +from pytorch3d import _C +from torch.autograd import Function +from torch.autograd.function import once_differentiable + + +_KNN = namedtuple("KNN", "dists idx knn") + + +class _knn_points(Function): + """ + Torch autograd Function wrapper for KNN C++/CUDA implementations. + """ + + @staticmethod + # pyre-fixme[14]: `forward` overrides method defined in `Function` inconsistently. + def forward( + ctx, + p1, + p2, + lengths1, + lengths2, + K, + version, + norm: int = 2, + return_sorted: bool = True, + ): + """ + K-Nearest neighbors on point clouds. + + Args: + p1: Tensor of shape (N, P1, D) giving a batch of N point clouds, each + containing up to P1 points of dimension D. + p2: Tensor of shape (N, P2, D) giving a batch of N point clouds, each + containing up to P2 points of dimension D. + lengths1: LongTensor of shape (N,) of values in the range [0, P1], giving the + length of each pointcloud in p1. Or None to indicate that every cloud has + length P1. + lengths2: LongTensor of shape (N,) of values in the range [0, P2], giving the + length of each pointcloud in p2. Or None to indicate that every cloud has + length P2. + K: Integer giving the number of nearest neighbors to return. + version: Which KNN implementation to use in the backend. If version=-1, + the correct implementation is selected based on the shapes of the inputs. + norm: (int) indicating the norm. Only supports 1 (for L1) and 2 (for L2). + return_sorted: (bool) whether to return the nearest neighbors sorted in + ascending order of distance. + + Returns: + p1_dists: Tensor of shape (N, P1, K) giving the squared distances to + the nearest neighbors. This is padded with zeros both where a cloud in p2 + has fewer than K points and where a cloud in p1 has fewer than P1 points. + + p1_idx: LongTensor of shape (N, P1, K) giving the indices of the + K nearest neighbors from points in p1 to points in p2. + Concretely, if `p1_idx[n, i, k] = j` then `p2[n, j]` is the k-th nearest + neighbors to `p1[n, i]` in `p2[n]`. This is padded with zeros both where a cloud + in p2 has fewer than K points and where a cloud in p1 has fewer than P1 points. + """ + if not ((norm == 1) or (norm == 2)): + raise ValueError("Support for 1 or 2 norm.") + + idx, dists = _C.knn_points_idx(p1, p2, lengths1, lengths2, norm, K, version) + + # sort KNN in ascending order if K > 1 + if K > 1 and return_sorted: + if lengths2.min() < K: + P1 = p1.shape[1] + mask = lengths2[:, None] <= torch.arange(K, device=dists.device)[None] + # mask has shape [N, K], true where dists irrelevant + mask = mask[:, None].expand(-1, P1, -1) + # mask has shape [N, P1, K], true where dists irrelevant + dists[mask] = float("inf") + dists, sort_idx = dists.sort(dim=2) + dists[mask] = 0 + else: + dists, sort_idx = dists.sort(dim=2) + idx = idx.gather(2, sort_idx) + + ctx.save_for_backward(p1, p2, lengths1, lengths2, idx) + ctx.mark_non_differentiable(idx) + ctx.norm = norm + return dists, idx + + @staticmethod + @once_differentiable + def backward(ctx, grad_dists, grad_idx): + p1, p2, lengths1, lengths2, idx = ctx.saved_tensors + norm = ctx.norm + # TODO(gkioxari) Change cast to floats once we add support for doubles. + if not (grad_dists.dtype == torch.float32): + grad_dists = grad_dists.float() + if not (p1.dtype == torch.float32): + p1 = p1.float() + if not (p2.dtype == torch.float32): + p2 = p2.float() + grad_p1, grad_p2 = _C.knn_points_backward( + p1, p2, lengths1, lengths2, idx, norm, grad_dists + ) + return grad_p1, grad_p2, None, None, None, None, None, None + + +def knn_points( + p1: torch.Tensor, + p2: torch.Tensor, + lengths1: Union[torch.Tensor, None] = None, + lengths2: Union[torch.Tensor, None] = None, + norm: int = 2, + K: int = 1, + version: int = -1, + return_nn: bool = False, + return_sorted: bool = True, +) -> _KNN: + """ + K-Nearest neighbors on point clouds. + + Args: + p1: Tensor of shape (N, P1, D) giving a batch of N point clouds, each + containing up to P1 points of dimension D. + p2: Tensor of shape (N, P2, D) giving a batch of N point clouds, each + containing up to P2 points of dimension D. + lengths1: LongTensor of shape (N,) of values in the range [0, P1], giving the + length of each pointcloud in p1. Or None to indicate that every cloud has + length P1. + lengths2: LongTensor of shape (N,) of values in the range [0, P2], giving the + length of each pointcloud in p2. Or None to indicate that every cloud has + length P2. + norm: Integer indicating the norm of the distance. Supports only 1 for L1, 2 for L2. + K: Integer giving the number of nearest neighbors to return. + version: Which KNN implementation to use in the backend. If version=-1, + the correct implementation is selected based on the shapes of the inputs. + return_nn: If set to True returns the K nearest neighbors in p2 for each point in p1. + return_sorted: (bool) whether to return the nearest neighbors sorted in + ascending order of distance. + + Returns: + dists: Tensor of shape (N, P1, K) giving the squared distances to + the nearest neighbors. This is padded with zeros both where a cloud in p2 + has fewer than K points and where a cloud in p1 has fewer than P1 points. + + idx: LongTensor of shape (N, P1, K) giving the indices of the + K nearest neighbors from points in p1 to points in p2. + Concretely, if `p1_idx[n, i, k] = j` then `p2[n, j]` is the k-th nearest + neighbors to `p1[n, i]` in `p2[n]`. This is padded with zeros both where a cloud + in p2 has fewer than K points and where a cloud in p1 has fewer than P1 + points. + + nn: Tensor of shape (N, P1, K, D) giving the K nearest neighbors in p2 for + each point in p1. Concretely, `p2_nn[n, i, k]` gives the k-th nearest neighbor + for `p1[n, i]`. Returned if `return_nn` is True. + The nearest neighbors are collected using `knn_gather` + + .. code-block:: + + p2_nn = knn_gather(p2, p1_idx, lengths2) + + which is a helper function that allows indexing any tensor of shape (N, P2, U) with + the indices `p1_idx` returned by `knn_points`. The output is a tensor + of shape (N, P1, K, U). + + """ + if p1.shape[0] != p2.shape[0]: + raise ValueError("pts1 and pts2 must have the same batch dimension.") + if p1.shape[2] != p2.shape[2]: + raise ValueError("pts1 and pts2 must have the same point dimension.") + + p1 = p1.contiguous() + p2 = p2.contiguous() + + P1 = p1.shape[1] + P2 = p2.shape[1] + + if lengths1 is None: + lengths1 = torch.full((p1.shape[0],), P1, dtype=torch.int64, device=p1.device) + if lengths2 is None: + lengths2 = torch.full((p1.shape[0],), P2, dtype=torch.int64, device=p1.device) + + p1_dists, p1_idx = _knn_points.apply( + p1, p2, lengths1, lengths2, K, version, norm, return_sorted + ) + + p2_nn = None + if return_nn: + p2_nn = knn_gather(p2, p1_idx, lengths2) + + return _KNN(dists=p1_dists, idx=p1_idx, knn=p2_nn if return_nn else None) + + +def knn_gather( + x: torch.Tensor, idx: torch.Tensor, lengths: Union[torch.Tensor, None] = None +): + """ + A helper function for knn that allows indexing a tensor x with the indices `idx` + returned by `knn_points`. + + For example, if `dists, idx = knn_points(p, x, lengths_p, lengths, K)` + where p is a tensor of shape (N, L, D) and x a tensor of shape (N, M, D), + then one can compute the K nearest neighbors of p with `p_nn = knn_gather(x, idx, lengths)`. + It can also be applied for any tensor x of shape (N, M, U) where U != D. + + Args: + x: Tensor of shape (N, M, U) containing U-dimensional features to + be gathered. + idx: LongTensor of shape (N, L, K) giving the indices returned by `knn_points`. + lengths: LongTensor of shape (N,) of values in the range [0, M], giving the + length of each example in the batch in x. Or None to indicate that every + example has length M. + Returns: + x_out: Tensor of shape (N, L, K, U) resulting from gathering the elements of x + with idx, s.t. `x_out[n, l, k] = x[n, idx[n, l, k]]`. + If `k > lengths[n]` then `x_out[n, l, k]` is filled with 0.0. + """ + N, M, U = x.shape + _N, L, K = idx.shape + + if N != _N: + raise ValueError("x and idx must have same batch dimension.") + + if lengths is None: + lengths = torch.full((x.shape[0],), M, dtype=torch.int64, device=x.device) + + idx_expanded = idx[:, :, :, None].expand(-1, -1, -1, U) + # idx_expanded has shape [N, L, K, U] + + x_out = x[:, :, None].expand(-1, -1, K, -1).gather(1, idx_expanded) + # p2_nn has shape [N, L, K, U] + + needs_mask = lengths.min() < K + if needs_mask: + # mask has shape [N, K], true where idx is irrelevant because + # there is less number of points in p2 than K + mask = lengths[:, None] <= torch.arange(K, device=x.device)[None] + + # expand mask to shape [N, L, K, U] + mask = mask[:, None].expand(-1, L, -1) + mask = mask[:, :, :, None].expand(-1, -1, -1, U) + x_out[mask] = 0.0 + + return x_out diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/laplacian_matrices.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/laplacian_matrices.py new file mode 100644 index 0000000000000000000000000000000000000000..6400923f385446767728a8d1b733b7955ee1391a --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/laplacian_matrices.py @@ -0,0 +1,182 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from typing import Tuple + +import torch + + +# ------------------------ Laplacian Matrices ------------------------ # +# This file contains implementations of differentiable laplacian matrices. +# These include +# 1) Standard Laplacian matrix +# 2) Cotangent Laplacian matrix +# 3) Norm Laplacian matrix +# -------------------------------------------------------------------- # + + +def laplacian(verts: torch.Tensor, edges: torch.Tensor) -> torch.Tensor: + """ + Computes the laplacian matrix. + The definition of the laplacian is + L[i, j] = -1 , if i == j + L[i, j] = 1 / deg(i) , if (i, j) is an edge + L[i, j] = 0 , otherwise + where deg(i) is the degree of the i-th vertex in the graph. + + Args: + verts: tensor of shape (V, 3) containing the vertices of the graph + edges: tensor of shape (E, 2) containing the vertex indices of each edge + Returns: + L: Sparse FloatTensor of shape (V, V) + """ + V = verts.shape[0] + + e0, e1 = edges.unbind(1) + + idx01 = torch.stack([e0, e1], dim=1) # (E, 2) + idx10 = torch.stack([e1, e0], dim=1) # (E, 2) + idx = torch.cat([idx01, idx10], dim=0).t() # (2, 2*E) + + # First, we construct the adjacency matrix, + # i.e. A[i, j] = 1 if (i,j) is an edge, or + # A[e0, e1] = 1 & A[e1, e0] = 1 + ones = torch.ones(idx.shape[1], dtype=torch.float32, device=verts.device) + # pyre-fixme[16]: Module `sparse` has no attribute `FloatTensor`. + A = torch.sparse.FloatTensor(idx, ones, (V, V)) + + # the sum of i-th row of A gives the degree of the i-th vertex + deg = torch.sparse.sum(A, dim=1).to_dense() + + # We construct the Laplacian matrix by adding the non diagonal values + # i.e. L[i, j] = 1 ./ deg(i) if (i, j) is an edge + deg0 = deg[e0] + # pyre-fixme[58]: `/` is not supported for operand types `float` and `Tensor`. + deg0 = torch.where(deg0 > 0.0, 1.0 / deg0, deg0) + deg1 = deg[e1] + # pyre-fixme[58]: `/` is not supported for operand types `float` and `Tensor`. + deg1 = torch.where(deg1 > 0.0, 1.0 / deg1, deg1) + val = torch.cat([deg0, deg1]) + # pyre-fixme[16]: Module `sparse` has no attribute `FloatTensor`. + L = torch.sparse.FloatTensor(idx, val, (V, V)) + + # Then we add the diagonal values L[i, i] = -1. + idx = torch.arange(V, device=verts.device) + idx = torch.stack([idx, idx], dim=0) + ones = torch.ones(idx.shape[1], dtype=torch.float32, device=verts.device) + # pyre-fixme[16]: Module `sparse` has no attribute `FloatTensor`. + L -= torch.sparse.FloatTensor(idx, ones, (V, V)) + + return L + + +def cot_laplacian( + verts: torch.Tensor, faces: torch.Tensor, eps: float = 1e-12 +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Returns the Laplacian matrix with cotangent weights and the inverse of the + face areas. + + Args: + verts: tensor of shape (V, 3) containing the vertices of the graph + faces: tensor of shape (F, 3) containing the vertex indices of each face + Returns: + 2-element tuple containing + - **L**: Sparse FloatTensor of shape (V,V) for the Laplacian matrix. + Here, L[i, j] = cot a_ij + cot b_ij iff (i, j) is an edge in meshes. + See the description above for more clarity. + - **inv_areas**: FloatTensor of shape (V,) containing the inverse of sum of + face areas containing each vertex + """ + V, F = verts.shape[0], faces.shape[0] + + face_verts = verts[faces] + v0, v1, v2 = face_verts[:, 0], face_verts[:, 1], face_verts[:, 2] + + # Side lengths of each triangle, of shape (sum(F_n),) + # A is the side opposite v1, B is opposite v2, and C is opposite v3 + A = (v1 - v2).norm(dim=1) + B = (v0 - v2).norm(dim=1) + C = (v0 - v1).norm(dim=1) + + # Area of each triangle (with Heron's formula); shape is (sum(F_n),) + s = 0.5 * (A + B + C) + # note that the area can be negative (close to 0) causing nans after sqrt() + # we clip it to a small positive value + # pyre-fixme[16]: `float` has no attribute `clamp_`. + area = (s * (s - A) * (s - B) * (s - C)).clamp_(min=eps).sqrt() + + # Compute cotangents of angles, of shape (sum(F_n), 3) + A2, B2, C2 = A * A, B * B, C * C + cota = (B2 + C2 - A2) / area + cotb = (A2 + C2 - B2) / area + cotc = (A2 + B2 - C2) / area + cot = torch.stack([cota, cotb, cotc], dim=1) + cot /= 4.0 + + # Construct a sparse matrix by basically doing: + # L[v1, v2] = cota + # L[v2, v0] = cotb + # L[v0, v1] = cotc + ii = faces[:, [1, 2, 0]] + jj = faces[:, [2, 0, 1]] + idx = torch.stack([ii, jj], dim=0).view(2, F * 3) + # pyre-fixme[16]: Module `sparse` has no attribute `FloatTensor`. + L = torch.sparse.FloatTensor(idx, cot.view(-1), (V, V)) + + # Make it symmetric; this means we are also setting + # L[v2, v1] = cota + # L[v0, v2] = cotb + # L[v1, v0] = cotc + L += L.t() + + # For each vertex, compute the sum of areas for triangles containing it. + idx = faces.view(-1) + inv_areas = torch.zeros(V, dtype=torch.float32, device=verts.device) + val = torch.stack([area] * 3, dim=1).view(-1) + inv_areas.scatter_add_(0, idx, val) + idx = inv_areas > 0 + # pyre-fixme[58]: `/` is not supported for operand types `float` and `Tensor`. + inv_areas[idx] = 1.0 / inv_areas[idx] + inv_areas = inv_areas.view(-1, 1) + + return L, inv_areas + + +def norm_laplacian( + verts: torch.Tensor, edges: torch.Tensor, eps: float = 1e-12 +) -> torch.Tensor: + """ + Norm laplacian computes a variant of the laplacian matrix which weights each + affinity with the normalized distance of the neighboring nodes. + More concretely, + L[i, j] = 1. / wij where wij = ||vi - vj|| if (vi, vj) are neighboring nodes + + Args: + verts: tensor of shape (V, 3) containing the vertices of the graph + edges: tensor of shape (E, 2) containing the vertex indices of each edge + Returns: + L: Sparse FloatTensor of shape (V, V) + """ + edge_verts = verts[edges] # (E, 2, 3) + v0, v1 = edge_verts[:, 0], edge_verts[:, 1] + + # Side lengths of each edge, of shape (E,) + w01 = 1.0 / ((v0 - v1).norm(dim=1) + eps) + + # Construct a sparse matrix by basically doing: + # L[v0, v1] = w01 + # L[v1, v0] = w01 + e01 = edges.t() # (2, E) + + V = verts.shape[0] + # pyre-fixme[16]: Module `sparse` has no attribute `FloatTensor`. + L = torch.sparse.FloatTensor(e01, w01, (V, V)) + L = L + L.t() + + return L diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/marching_cubes.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/marching_cubes.py new file mode 100644 index 0000000000000000000000000000000000000000..17236857d607a17270120606288451f51cb999a7 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/marching_cubes.py @@ -0,0 +1,305 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from typing import List, Optional, Tuple + +import torch +from pytorch3d import _C +from pytorch3d.ops.marching_cubes_data import EDGE_TO_VERTICES, FACE_TABLE, INDEX +from pytorch3d.transforms import Translate +from torch.autograd import Function + + +EPS = 0.00001 + + +class Cube: + def __init__( + self, + bfl_v: Tuple[int, int, int], + volume: torch.Tensor, + isolevel: float, + ) -> None: + """ + Initializes a cube given the bottom front left vertex coordinate + and computes the cube configuration given vertex values and isolevel. + + Edge and vertex convention: + + v4_______e4____________v5 + /| /| + / | / | + e7/ | e5/ | + /___|______e6_________/ | + v7| | |v6 |e9 + | | | | + | |e8 |e10| + e11| | | | + | |______e0_________|___| + | / v0(bfl_v) | |v1 + | / | / + | /e3 | /e1 + |/_____________________|/ + v3 e2 v2 + + Args: + bfl_vertex: a tuple of size 3 corresponding to the bottom front left vertex + of the cube in (x, y, z) format + volume: the 3D scalar data + isolevel: the isosurface value used as a threshold for determining whether a point + is inside/outside the volume + """ + x, y, z = bfl_v + self.x, self.y, self.z = x, y, z + self.bfl_v = bfl_v + self.verts = [ + [x + (v & 1), y + (v >> 1 & 1), z + (v >> 2 & 1)] for v in range(8) + ] # vertex position (x, y, z) for v0-v1-v4-v5-v3-v2-v7-v6 + + # Calculates cube configuration index given values of the cube vertices + self.cube_index = 0 + for i in range(8): + v = self.verts[INDEX[i]] + value = volume[v[2]][v[1]][v[0]] + if value < isolevel: + self.cube_index |= 1 << i + + def get_vpair_from_edge(self, edge: int, W: int, H: int) -> Tuple[int, int]: + """ + Get a tuple of global vertex ID from a local edge ID + Global vertex ID is calculated as (x + dx) + (y + dy) * W + (z + dz) * W * H + + Args: + edge: local edge ID in the cube + bfl_vertex: bottom-front-left coordinate of the cube + + Returns: + a pair of global vertex ID + """ + v1, v2 = EDGE_TO_VERTICES[edge] # two end-points on the edge + v1_id = self.verts[v1][0] + self.verts[v1][1] * W + self.verts[v1][2] * W * H + v2_id = self.verts[v2][0] + self.verts[v2][1] * W + self.verts[v2][2] * W * H + return (v1_id, v2_id) + + def vert_interp( + self, + isolevel: float, + edge: int, + vol: torch.Tensor, + ) -> List: + """ + Linearly interpolate a vertex where an isosurface cuts an edge + between the two endpoint vertices, based on their values + + Args: + isolevel: the isosurface value to use as the threshold to determine + whether points are within a volume. + edge: edge (ID) to interpolate + cube: current cube vertices + vol: 3D scalar field + + Returns: + interpolated vertex: position of the interpolated vertex on the edge + """ + v1, v2 = EDGE_TO_VERTICES[edge] + p1, p2 = self.verts[v1], self.verts[v2] + val1, val2 = ( + vol[p1[2]][p1[1]][p1[0]], + vol[p2[2]][p2[1]][p2[0]], + ) + point = None + if abs(isolevel - val1) < EPS: + point = p1 + elif abs(isolevel - val2) < EPS: + point = p2 + elif abs(val1 - val2) < EPS: + point = p1 + + if point is None: + mu = (isolevel - val1) / (val2 - val1) + x1, y1, z1 = p1 + x2, y2, z2 = p2 + x = x1 + mu * (x2 - x1) + y = y1 + mu * (y2 - y1) + z = z1 + mu * (z2 - z1) + else: + x, y, z = point + return [x, y, z] + + +def marching_cubes_naive( + vol_batch: torch.Tensor, + isolevel: Optional[float] = None, + return_local_coords: bool = True, +) -> Tuple[List[torch.Tensor], List[torch.Tensor]]: + """ + Runs the classic marching cubes algorithm, iterating over + the coordinates of the volume and using a given isolevel + for determining intersected edges of cubes. + Returns vertices and faces of the obtained mesh. + This operation is non-differentiable. + + Args: + vol_batch: a Tensor of size (N, D, H, W) corresponding to + a batch of 3D scalar fields + isolevel: the isosurface value to use as the threshold to determine + whether points are within a volume. If None, then the average of the + maximum and minimum value of the scalar field will be used. + return_local_coords: bool. If True the output vertices will be in local coordinates in + the range [-1, 1] x [-1, 1] x [-1, 1]. If False they will be in the range + [0, W-1] x [0, H-1] x [0, D-1] + Returns: + verts: [{V_0}, {V_1}, ...] List of N sets of vertices of shape (|V_i|, 3) in FloatTensor + faces: [{F_0}, {F_1}, ...] List of N sets of faces of shape (|F_i|, 3) in LongTensors + """ + batched_verts, batched_faces = [], [] + D, H, W = vol_batch.shape[1:] + + # each edge is represented with its two endpoints (represented with global id) + for i in range(len(vol_batch)): + vol = vol_batch[i] + thresh = ((vol.max() + vol.min()) / 2).item() if isolevel is None else isolevel + vpair_to_edge = {} # maps from tuple of edge endpoints to edge_id + edge_id_to_v = {} # maps from edge ID to vertex position + uniq_edge_id = {} # unique edge IDs + verts = [] # store vertex positions + faces = [] # store face indices + # enumerate each cell in the 3d grid + for z in range(0, D - 1): + for y in range(0, H - 1): + for x in range(0, W - 1): + cube = Cube((x, y, z), vol, thresh) + edge_indices = FACE_TABLE[cube.cube_index] + # cube is entirely in/out of the surface + if len(edge_indices) == 0: + continue + + # gather mesh vertices/faces by processing each cube + interp_points = [[0.0, 0.0, 0.0]] * 12 + # triangle vertex IDs and positions + tri = [] + ps = [] + for i, edge in enumerate(edge_indices): + interp_points[edge] = cube.vert_interp(thresh, edge, vol) + + # Bind interpolated vertex with a global edge_id, which + # is represented by a pair of vertex ids (v1_id, v2_id) + # corresponding to a local edge. + (v1_id, v2_id) = cube.get_vpair_from_edge(edge, W, H) + edge_id = vpair_to_edge.setdefault( + (v1_id, v2_id), len(vpair_to_edge) + ) + tri.append(edge_id) + ps.append(interp_points[edge]) + # when the isolevel are the same as the edge endpoints, the interploated + # vertices can share the same values, and lead to degenerate triangles. + if ( + (i + 1) % 3 == 0 + and ps[0] != ps[1] + and ps[1] != ps[2] + and ps[2] != ps[0] + ): + for j, edge_id in enumerate(tri): + edge_id_to_v[edge_id] = ps[j] + if edge_id not in uniq_edge_id: + uniq_edge_id[edge_id] = len(verts) + verts.append(edge_id_to_v[edge_id]) + faces.append([uniq_edge_id[tri[j]] for j in range(3)]) + tri = [] + ps = [] + + if len(faces) > 0 and len(verts) > 0: + verts = torch.tensor(verts, dtype=vol.dtype) + # Convert from world coordinates ([0, D-1], [0, H-1], [0, W-1]) to + # local coordinates in the range [-1, 1] + if return_local_coords: + verts = ( + Translate(x=+1.0, y=+1.0, z=+1.0, device=vol_batch.device) + .scale((vol_batch.new_tensor([W, H, D])[None] - 1) * 0.5) + .inverse() + ).transform_points(verts[None])[0] + batched_verts.append(verts) + batched_faces.append(torch.tensor(faces, dtype=torch.int64)) + else: + batched_verts.append([]) + batched_faces.append([]) + return batched_verts, batched_faces + + +######################################## +# Marching Cubes Implementation in C++/Cuda +######################################## +class _marching_cubes(Function): + """ + Torch Function wrapper for marching_cubes implementation. + This function is not differentiable. An autograd wrapper is used + to ensure an error if user tries to get gradients. + """ + + @staticmethod + def forward(ctx, vol, isolevel): + verts, faces, ids = _C.marching_cubes(vol, isolevel) + return verts, faces, ids + + @staticmethod + def backward(ctx, grad_verts, grad_faces): + raise ValueError("marching_cubes backward is not supported") + + +def marching_cubes( + vol_batch: torch.Tensor, + isolevel: Optional[float] = None, + return_local_coords: bool = True, +) -> Tuple[List[torch.Tensor], List[torch.Tensor]]: + """ + Run marching cubes over a volume scalar field with a designated isolevel. + Returns vertices and faces of the obtained mesh. + This operation is non-differentiable. + + Args: + vol_batch: a Tensor of size (N, D, H, W) corresponding to + a batch of 3D scalar fields + isolevel: float used as threshold to determine if a point is inside/outside + the volume. If None, then the average of the maximum and minimum value + of the scalar field is used. + return_local_coords: bool. If True the output vertices will be in local coordinates in + the range [-1, 1] x [-1, 1] x [-1, 1]. If False they will be in the range + [0, W-1] x [0, H-1] x [0, D-1] + + Returns: + verts: [{V_0}, {V_1}, ...] List of N sets of vertices of shape (|V_i|, 3) in FloatTensor + faces: [{F_0}, {F_1}, ...] List of N sets of faces of shape (|F_i|, 3) in LongTensors + """ + batched_verts, batched_faces = [], [] + D, H, W = vol_batch.shape[1:] + for i in range(len(vol_batch)): + vol = vol_batch[i] + thresh = ((vol.max() + vol.min()) / 2).item() if isolevel is None else isolevel + verts, faces, ids = _marching_cubes.apply(vol, thresh) + if len(faces) > 0 and len(verts) > 0: + # Convert from world coordinates ([0, D-1], [0, H-1], [0, W-1]) to + # local coordinates in the range [-1, 1] + if return_local_coords: + verts = ( + Translate(x=+1.0, y=+1.0, z=+1.0, device=vol.device) + .scale((vol.new_tensor([W, H, D])[None] - 1) * 0.5) + .inverse() + ).transform_points(verts[None])[0] + # deduplication for cuda + if vol.is_cuda: + unique_ids, inverse_idx = torch.unique(ids, return_inverse=True) + verts_ = verts.new_zeros(unique_ids.shape[0], 3) + verts_[inverse_idx] = verts + verts = verts_ + faces = inverse_idx[faces] + batched_verts.append(verts) + batched_faces.append(faces) + else: + batched_verts.append([]) + batched_faces.append([]) + return batched_verts, batched_faces diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/marching_cubes_data.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/marching_cubes_data.py new file mode 100644 index 0000000000000000000000000000000000000000..802377e0795a980ca539e554d40ecd11ef4b3902 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/marching_cubes_data.py @@ -0,0 +1,291 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +# Maps each edge (by index) to the corresponding cube vertices +EDGE_TO_VERTICES = [ + [0, 1], + [1, 5], + [4, 5], + [0, 4], + [2, 3], + [3, 7], + [6, 7], + [2, 6], + [0, 2], + [1, 3], + [5, 7], + [4, 6], +] + +# A list of lists mapping a cube_index (a given configuration) +# to a list of faces corresponding to that configuration. Each face is represented +# by 3 consecutive numbers. A configuration will at most have 5 faces. +# +# Table taken from http://paulbourke.net/geometry/polygonise/ +FACE_TABLE = [ + [], + [0, 8, 3], + [0, 1, 9], + [1, 8, 3, 9, 8, 1], + [1, 2, 10], + [0, 8, 3, 1, 2, 10], + [9, 2, 10, 0, 2, 9], + [2, 8, 3, 2, 10, 8, 10, 9, 8], + [3, 11, 2], + [0, 11, 2, 8, 11, 0], + [1, 9, 0, 2, 3, 11], + [1, 11, 2, 1, 9, 11, 9, 8, 11], + [3, 10, 1, 11, 10, 3], + [0, 10, 1, 0, 8, 10, 8, 11, 10], + [3, 9, 0, 3, 11, 9, 11, 10, 9], + [9, 8, 10, 10, 8, 11], + [4, 7, 8], + [4, 3, 0, 7, 3, 4], + [0, 1, 9, 8, 4, 7], + [4, 1, 9, 4, 7, 1, 7, 3, 1], + [1, 2, 10, 8, 4, 7], + [3, 4, 7, 3, 0, 4, 1, 2, 10], + [9, 2, 10, 9, 0, 2, 8, 4, 7], + [2, 10, 9, 2, 9, 7, 2, 7, 3, 7, 9, 4], + [8, 4, 7, 3, 11, 2], + [11, 4, 7, 11, 2, 4, 2, 0, 4], + [9, 0, 1, 8, 4, 7, 2, 3, 11], + [4, 7, 11, 9, 4, 11, 9, 11, 2, 9, 2, 1], + [3, 10, 1, 3, 11, 10, 7, 8, 4], + [1, 11, 10, 1, 4, 11, 1, 0, 4, 7, 11, 4], + [4, 7, 8, 9, 0, 11, 9, 11, 10, 11, 0, 3], + [4, 7, 11, 4, 11, 9, 9, 11, 10], + [9, 5, 4], + [9, 5, 4, 0, 8, 3], + [0, 5, 4, 1, 5, 0], + [8, 5, 4, 8, 3, 5, 3, 1, 5], + [1, 2, 10, 9, 5, 4], + [3, 0, 8, 1, 2, 10, 4, 9, 5], + [5, 2, 10, 5, 4, 2, 4, 0, 2], + [2, 10, 5, 3, 2, 5, 3, 5, 4, 3, 4, 8], + [9, 5, 4, 2, 3, 11], + [0, 11, 2, 0, 8, 11, 4, 9, 5], + [0, 5, 4, 0, 1, 5, 2, 3, 11], + [2, 1, 5, 2, 5, 8, 2, 8, 11, 4, 8, 5], + [10, 3, 11, 10, 1, 3, 9, 5, 4], + [4, 9, 5, 0, 8, 1, 8, 10, 1, 8, 11, 10], + [5, 4, 0, 5, 0, 11, 5, 11, 10, 11, 0, 3], + [5, 4, 8, 5, 8, 10, 10, 8, 11], + [9, 7, 8, 5, 7, 9], + [9, 3, 0, 9, 5, 3, 5, 7, 3], + [0, 7, 8, 0, 1, 7, 1, 5, 7], + [1, 5, 3, 3, 5, 7], + [9, 7, 8, 9, 5, 7, 10, 1, 2], + [10, 1, 2, 9, 5, 0, 5, 3, 0, 5, 7, 3], + [8, 0, 2, 8, 2, 5, 8, 5, 7, 10, 5, 2], + [2, 10, 5, 2, 5, 3, 3, 5, 7], + [7, 9, 5, 7, 8, 9, 3, 11, 2], + [9, 5, 7, 9, 7, 2, 9, 2, 0, 2, 7, 11], + [2, 3, 11, 0, 1, 8, 1, 7, 8, 1, 5, 7], + [11, 2, 1, 11, 1, 7, 7, 1, 5], + [9, 5, 8, 8, 5, 7, 10, 1, 3, 10, 3, 11], + [5, 7, 0, 5, 0, 9, 7, 11, 0, 1, 0, 10, 11, 10, 0], + [11, 10, 0, 11, 0, 3, 10, 5, 0, 8, 0, 7, 5, 7, 0], + [11, 10, 5, 7, 11, 5], + [10, 6, 5], + [0, 8, 3, 5, 10, 6], + [9, 0, 1, 5, 10, 6], + [1, 8, 3, 1, 9, 8, 5, 10, 6], + [1, 6, 5, 2, 6, 1], + [1, 6, 5, 1, 2, 6, 3, 0, 8], + [9, 6, 5, 9, 0, 6, 0, 2, 6], + [5, 9, 8, 5, 8, 2, 5, 2, 6, 3, 2, 8], + [2, 3, 11, 10, 6, 5], + [11, 0, 8, 11, 2, 0, 10, 6, 5], + [0, 1, 9, 2, 3, 11, 5, 10, 6], + [5, 10, 6, 1, 9, 2, 9, 11, 2, 9, 8, 11], + [6, 3, 11, 6, 5, 3, 5, 1, 3], + [0, 8, 11, 0, 11, 5, 0, 5, 1, 5, 11, 6], + [3, 11, 6, 0, 3, 6, 0, 6, 5, 0, 5, 9], + [6, 5, 9, 6, 9, 11, 11, 9, 8], + [5, 10, 6, 4, 7, 8], + [4, 3, 0, 4, 7, 3, 6, 5, 10], + [1, 9, 0, 5, 10, 6, 8, 4, 7], + [10, 6, 5, 1, 9, 7, 1, 7, 3, 7, 9, 4], + [6, 1, 2, 6, 5, 1, 4, 7, 8], + [1, 2, 5, 5, 2, 6, 3, 0, 4, 3, 4, 7], + [8, 4, 7, 9, 0, 5, 0, 6, 5, 0, 2, 6], + [7, 3, 9, 7, 9, 4, 3, 2, 9, 5, 9, 6, 2, 6, 9], + [3, 11, 2, 7, 8, 4, 10, 6, 5], + [5, 10, 6, 4, 7, 2, 4, 2, 0, 2, 7, 11], + [0, 1, 9, 4, 7, 8, 2, 3, 11, 5, 10, 6], + [9, 2, 1, 9, 11, 2, 9, 4, 11, 7, 11, 4, 5, 10, 6], + [8, 4, 7, 3, 11, 5, 3, 5, 1, 5, 11, 6], + [5, 1, 11, 5, 11, 6, 1, 0, 11, 7, 11, 4, 0, 4, 11], + [0, 5, 9, 0, 6, 5, 0, 3, 6, 11, 6, 3, 8, 4, 7], + [6, 5, 9, 6, 9, 11, 4, 7, 9, 7, 11, 9], + [10, 4, 9, 6, 4, 10], + [4, 10, 6, 4, 9, 10, 0, 8, 3], + [10, 0, 1, 10, 6, 0, 6, 4, 0], + [8, 3, 1, 8, 1, 6, 8, 6, 4, 6, 1, 10], + [1, 4, 9, 1, 2, 4, 2, 6, 4], + [3, 0, 8, 1, 2, 9, 2, 4, 9, 2, 6, 4], + [0, 2, 4, 4, 2, 6], + [8, 3, 2, 8, 2, 4, 4, 2, 6], + [10, 4, 9, 10, 6, 4, 11, 2, 3], + [0, 8, 2, 2, 8, 11, 4, 9, 10, 4, 10, 6], + [3, 11, 2, 0, 1, 6, 0, 6, 4, 6, 1, 10], + [6, 4, 1, 6, 1, 10, 4, 8, 1, 2, 1, 11, 8, 11, 1], + [9, 6, 4, 9, 3, 6, 9, 1, 3, 11, 6, 3], + [8, 11, 1, 8, 1, 0, 11, 6, 1, 9, 1, 4, 6, 4, 1], + [3, 11, 6, 3, 6, 0, 0, 6, 4], + [6, 4, 8, 11, 6, 8], + [7, 10, 6, 7, 8, 10, 8, 9, 10], + [0, 7, 3, 0, 10, 7, 0, 9, 10, 6, 7, 10], + [10, 6, 7, 1, 10, 7, 1, 7, 8, 1, 8, 0], + [10, 6, 7, 10, 7, 1, 1, 7, 3], + [1, 2, 6, 1, 6, 8, 1, 8, 9, 8, 6, 7], + [2, 6, 9, 2, 9, 1, 6, 7, 9, 0, 9, 3, 7, 3, 9], + [7, 8, 0, 7, 0, 6, 6, 0, 2], + [7, 3, 2, 6, 7, 2], + [2, 3, 11, 10, 6, 8, 10, 8, 9, 8, 6, 7], + [2, 0, 7, 2, 7, 11, 0, 9, 7, 6, 7, 10, 9, 10, 7], + [1, 8, 0, 1, 7, 8, 1, 10, 7, 6, 7, 10, 2, 3, 11], + [11, 2, 1, 11, 1, 7, 10, 6, 1, 6, 7, 1], + [8, 9, 6, 8, 6, 7, 9, 1, 6, 11, 6, 3, 1, 3, 6], + [0, 9, 1, 11, 6, 7], + [7, 8, 0, 7, 0, 6, 3, 11, 0, 11, 6, 0], + [7, 11, 6], + [7, 6, 11], + [3, 0, 8, 11, 7, 6], + [0, 1, 9, 11, 7, 6], + [8, 1, 9, 8, 3, 1, 11, 7, 6], + [10, 1, 2, 6, 11, 7], + [1, 2, 10, 3, 0, 8, 6, 11, 7], + [2, 9, 0, 2, 10, 9, 6, 11, 7], + [6, 11, 7, 2, 10, 3, 10, 8, 3, 10, 9, 8], + [7, 2, 3, 6, 2, 7], + [7, 0, 8, 7, 6, 0, 6, 2, 0], + [2, 7, 6, 2, 3, 7, 0, 1, 9], + [1, 6, 2, 1, 8, 6, 1, 9, 8, 8, 7, 6], + [10, 7, 6, 10, 1, 7, 1, 3, 7], + [10, 7, 6, 1, 7, 10, 1, 8, 7, 1, 0, 8], + [0, 3, 7, 0, 7, 10, 0, 10, 9, 6, 10, 7], + [7, 6, 10, 7, 10, 8, 8, 10, 9], + [6, 8, 4, 11, 8, 6], + [3, 6, 11, 3, 0, 6, 0, 4, 6], + [8, 6, 11, 8, 4, 6, 9, 0, 1], + [9, 4, 6, 9, 6, 3, 9, 3, 1, 11, 3, 6], + [6, 8, 4, 6, 11, 8, 2, 10, 1], + [1, 2, 10, 3, 0, 11, 0, 6, 11, 0, 4, 6], + [4, 11, 8, 4, 6, 11, 0, 2, 9, 2, 10, 9], + [10, 9, 3, 10, 3, 2, 9, 4, 3, 11, 3, 6, 4, 6, 3], + [8, 2, 3, 8, 4, 2, 4, 6, 2], + [0, 4, 2, 4, 6, 2], + [1, 9, 0, 2, 3, 4, 2, 4, 6, 4, 3, 8], + [1, 9, 4, 1, 4, 2, 2, 4, 6], + [8, 1, 3, 8, 6, 1, 8, 4, 6, 6, 10, 1], + [10, 1, 0, 10, 0, 6, 6, 0, 4], + [4, 6, 3, 4, 3, 8, 6, 10, 3, 0, 3, 9, 10, 9, 3], + [10, 9, 4, 6, 10, 4], + [4, 9, 5, 7, 6, 11], + [0, 8, 3, 4, 9, 5, 11, 7, 6], + [5, 0, 1, 5, 4, 0, 7, 6, 11], + [11, 7, 6, 8, 3, 4, 3, 5, 4, 3, 1, 5], + [9, 5, 4, 10, 1, 2, 7, 6, 11], + [6, 11, 7, 1, 2, 10, 0, 8, 3, 4, 9, 5], + [7, 6, 11, 5, 4, 10, 4, 2, 10, 4, 0, 2], + [3, 4, 8, 3, 5, 4, 3, 2, 5, 10, 5, 2, 11, 7, 6], + [7, 2, 3, 7, 6, 2, 5, 4, 9], + [9, 5, 4, 0, 8, 6, 0, 6, 2, 6, 8, 7], + [3, 6, 2, 3, 7, 6, 1, 5, 0, 5, 4, 0], + [6, 2, 8, 6, 8, 7, 2, 1, 8, 4, 8, 5, 1, 5, 8], + [9, 5, 4, 10, 1, 6, 1, 7, 6, 1, 3, 7], + [1, 6, 10, 1, 7, 6, 1, 0, 7, 8, 7, 0, 9, 5, 4], + [4, 0, 10, 4, 10, 5, 0, 3, 10, 6, 10, 7, 3, 7, 10], + [7, 6, 10, 7, 10, 8, 5, 4, 10, 4, 8, 10], + [6, 9, 5, 6, 11, 9, 11, 8, 9], + [3, 6, 11, 0, 6, 3, 0, 5, 6, 0, 9, 5], + [0, 11, 8, 0, 5, 11, 0, 1, 5, 5, 6, 11], + [6, 11, 3, 6, 3, 5, 5, 3, 1], + [1, 2, 10, 9, 5, 11, 9, 11, 8, 11, 5, 6], + [0, 11, 3, 0, 6, 11, 0, 9, 6, 5, 6, 9, 1, 2, 10], + [11, 8, 5, 11, 5, 6, 8, 0, 5, 10, 5, 2, 0, 2, 5], + [6, 11, 3, 6, 3, 5, 2, 10, 3, 10, 5, 3], + [5, 8, 9, 5, 2, 8, 5, 6, 2, 3, 8, 2], + [9, 5, 6, 9, 6, 0, 0, 6, 2], + [1, 5, 8, 1, 8, 0, 5, 6, 8, 3, 8, 2, 6, 2, 8], + [1, 5, 6, 2, 1, 6], + [1, 3, 6, 1, 6, 10, 3, 8, 6, 5, 6, 9, 8, 9, 6], + [10, 1, 0, 10, 0, 6, 9, 5, 0, 5, 6, 0], + [0, 3, 8, 5, 6, 10], + [10, 5, 6], + [11, 5, 10, 7, 5, 11], + [11, 5, 10, 11, 7, 5, 8, 3, 0], + [5, 11, 7, 5, 10, 11, 1, 9, 0], + [10, 7, 5, 10, 11, 7, 9, 8, 1, 8, 3, 1], + [11, 1, 2, 11, 7, 1, 7, 5, 1], + [0, 8, 3, 1, 2, 7, 1, 7, 5, 7, 2, 11], + [9, 7, 5, 9, 2, 7, 9, 0, 2, 2, 11, 7], + [7, 5, 2, 7, 2, 11, 5, 9, 2, 3, 2, 8, 9, 8, 2], + [2, 5, 10, 2, 3, 5, 3, 7, 5], + [8, 2, 0, 8, 5, 2, 8, 7, 5, 10, 2, 5], + [9, 0, 1, 5, 10, 3, 5, 3, 7, 3, 10, 2], + [9, 8, 2, 9, 2, 1, 8, 7, 2, 10, 2, 5, 7, 5, 2], + [1, 3, 5, 3, 7, 5], + [0, 8, 7, 0, 7, 1, 1, 7, 5], + [9, 0, 3, 9, 3, 5, 5, 3, 7], + [9, 8, 7, 5, 9, 7], + [5, 8, 4, 5, 10, 8, 10, 11, 8], + [5, 0, 4, 5, 11, 0, 5, 10, 11, 11, 3, 0], + [0, 1, 9, 8, 4, 10, 8, 10, 11, 10, 4, 5], + [10, 11, 4, 10, 4, 5, 11, 3, 4, 9, 4, 1, 3, 1, 4], + [2, 5, 1, 2, 8, 5, 2, 11, 8, 4, 5, 8], + [0, 4, 11, 0, 11, 3, 4, 5, 11, 2, 11, 1, 5, 1, 11], + [0, 2, 5, 0, 5, 9, 2, 11, 5, 4, 5, 8, 11, 8, 5], + [9, 4, 5, 2, 11, 3], + [2, 5, 10, 3, 5, 2, 3, 4, 5, 3, 8, 4], + [5, 10, 2, 5, 2, 4, 4, 2, 0], + [3, 10, 2, 3, 5, 10, 3, 8, 5, 4, 5, 8, 0, 1, 9], + [5, 10, 2, 5, 2, 4, 1, 9, 2, 9, 4, 2], + [8, 4, 5, 8, 5, 3, 3, 5, 1], + [0, 4, 5, 1, 0, 5], + [8, 4, 5, 8, 5, 3, 9, 0, 5, 0, 3, 5], + [9, 4, 5], + [4, 11, 7, 4, 9, 11, 9, 10, 11], + [0, 8, 3, 4, 9, 7, 9, 11, 7, 9, 10, 11], + [1, 10, 11, 1, 11, 4, 1, 4, 0, 7, 4, 11], + [3, 1, 4, 3, 4, 8, 1, 10, 4, 7, 4, 11, 10, 11, 4], + [4, 11, 7, 9, 11, 4, 9, 2, 11, 9, 1, 2], + [9, 7, 4, 9, 11, 7, 9, 1, 11, 2, 11, 1, 0, 8, 3], + [11, 7, 4, 11, 4, 2, 2, 4, 0], + [11, 7, 4, 11, 4, 2, 8, 3, 4, 3, 2, 4], + [2, 9, 10, 2, 7, 9, 2, 3, 7, 7, 4, 9], + [9, 10, 7, 9, 7, 4, 10, 2, 7, 8, 7, 0, 2, 0, 7], + [3, 7, 10, 3, 10, 2, 7, 4, 10, 1, 10, 0, 4, 0, 10], + [1, 10, 2, 8, 7, 4], + [4, 9, 1, 4, 1, 7, 7, 1, 3], + [4, 9, 1, 4, 1, 7, 0, 8, 1, 8, 7, 1], + [4, 0, 3, 7, 4, 3], + [4, 8, 7], + [9, 10, 8, 10, 11, 8], + [3, 0, 9, 3, 9, 11, 11, 9, 10], + [0, 1, 10, 0, 10, 8, 8, 10, 11], + [3, 1, 10, 11, 3, 10], + [1, 2, 11, 1, 11, 9, 9, 11, 8], + [3, 0, 9, 3, 9, 11, 1, 2, 9, 2, 11, 9], + [0, 2, 11, 8, 0, 11], + [3, 2, 11], + [2, 3, 8, 2, 8, 10, 10, 8, 9], + [9, 10, 2, 0, 9, 2], + [2, 3, 8, 2, 8, 10, 0, 1, 8, 1, 10, 8], + [1, 10, 2], + [1, 3, 8, 9, 1, 8], + [0, 9, 1], + [0, 3, 8], + [], +] + +# mapping from 0-7 to v0-v7 in cube.vertices +INDEX = [0, 1, 5, 4, 2, 3, 7, 6] diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/mesh_face_areas_normals.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/mesh_face_areas_normals.py new file mode 100644 index 0000000000000000000000000000000000000000..ea91f482cf69eed491deabb3c940537d9d2efa48 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/mesh_face_areas_normals.py @@ -0,0 +1,68 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import torch +from pytorch3d import _C +from torch.autograd import Function +from torch.autograd.function import once_differentiable + + +class _MeshFaceAreasNormals(Function): + """ + Torch autograd Function wrapper for face areas & normals C++/CUDA implementations. + """ + + @staticmethod + def forward(ctx, verts, faces): + """ + Args: + ctx: Context object used to calculate gradients. + verts: FloatTensor of shape (V, 3), representing the packed + batch verts tensor. + faces: LongTensor of shape (F, 3), representing the packed + batch faces tensor + Returns: + areas: FloatTensor of shape (F,) with the areas of each face + normals: FloatTensor of shape (F,3) with the normals of each face + """ + if not (verts.dim() == 2): + raise ValueError("verts need to be of shape Vx3.") + if not (verts.shape[1] == 3): + raise ValueError("verts need to be of shape Vx3.") + if not (faces.dim() == 2): + raise ValueError("faces need to be of shape Fx3.") + if not (faces.shape[1] == 3): + raise ValueError("faces need to be of shape Fx3.") + if not (faces.dtype == torch.int64): + raise ValueError("faces need to be of type torch.int64.") + # TODO(gkioxari) Change cast to floats once we add support for doubles. + if not (verts.dtype == torch.float32): + verts = verts.float() + + ctx.save_for_backward(verts, faces) + areas, normals = _C.face_areas_normals_forward(verts, faces) + return areas, normals + + @staticmethod + @once_differentiable + def backward(ctx, grad_areas, grad_normals): + grad_areas = grad_areas.contiguous() + grad_normals = grad_normals.contiguous() + verts, faces = ctx.saved_tensors + # TODO(gkioxari) Change cast to floats once we add support for doubles. + if not (grad_areas.dtype == torch.float32): + grad_areas = grad_areas.float() + if not (grad_normals.dtype == torch.float32): + grad_normals = grad_normals.float() + grad_verts = _C.face_areas_normals_backward( + grad_areas, grad_normals, verts, faces + ) + return grad_verts, None + + +mesh_face_areas_normals = _MeshFaceAreasNormals.apply diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/mesh_filtering.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/mesh_filtering.py new file mode 100644 index 0000000000000000000000000000000000000000..d8f29431c6b6843411a51f14a69fd8ceac4f165f --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/mesh_filtering.py @@ -0,0 +1,62 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import torch +from pytorch3d.ops import norm_laplacian +from pytorch3d.structures import Meshes, utils as struct_utils + + +# ------------------------ Mesh Smoothing ------------------------ # +# This file contains differentiable operators to filter meshes +# The ops include +# 1) Taubin Smoothing +# TODO(gkioxari) add more! :) +# ---------------------------------------------------------------- # + + +# ----------------------- Taubin Smoothing ----------------------- # + + +def taubin_smoothing( + meshes: Meshes, lambd: float = 0.53, mu: float = -0.53, num_iter: int = 10 +) -> Meshes: + """ + Taubin smoothing [1] is an iterative smoothing operator for meshes. + At each iteration + verts := (1 - λ) * verts + λ * L * verts + verts := (1 - μ) * verts + μ * L * verts + + This function returns a new mesh with smoothed vertices. + Args: + meshes: Meshes input to be smoothed + lambd, mu: float parameters for Taubin smoothing, + lambd > 0, mu < 0 + num_iter: number of iterations to execute smoothing + Returns: + mesh: Smoothed input Meshes + + [1] Curve and Surface Smoothing without Shrinkage, + Gabriel Taubin, ICCV 1997 + """ + verts = meshes.verts_packed() # V x 3 + edges = meshes.edges_packed() # E x 3 + + for _ in range(num_iter): + L = norm_laplacian(verts, edges) + total_weight = torch.sparse.sum(L, dim=1).to_dense().view(-1, 1) + verts = (1 - lambd) * verts + lambd * torch.mm(L, verts) / total_weight + + L = norm_laplacian(verts, edges) + total_weight = torch.sparse.sum(L, dim=1).to_dense().view(-1, 1) + verts = (1 - mu) * verts + mu * torch.mm(L, verts) / total_weight + + verts_list = struct_utils.packed_to_list( + verts, meshes.num_verts_per_mesh().tolist() + ) + mesh = Meshes(verts=list(verts_list), faces=meshes.faces_list()) + return mesh diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/packed_to_padded.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/packed_to_padded.py new file mode 100644 index 0000000000000000000000000000000000000000..fd40bcc43e8ea0fa72b8a0f116cab0009967671d --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/packed_to_padded.py @@ -0,0 +1,198 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import torch +from pytorch3d import _C +from torch.autograd import Function +from torch.autograd.function import once_differentiable + + +class _PackedToPadded(Function): + """ + Torch autograd Function wrapper for packed_to_padded C++/CUDA implementations. + """ + + @staticmethod + def forward(ctx, inputs, first_idxs, max_size): + """ + Args: + ctx: Context object used to calculate gradients. + inputs: FloatTensor of shape (F, D), representing the packed batch tensor. + e.g. areas for faces in a batch of meshes. + first_idxs: LongTensor of shape (N,) where N is the number of + elements in the batch and `first_idxs[i] = f` + means that the inputs for batch element i begin at `inputs[f]`. + max_size: Max length of an element in the batch. + + Returns: + inputs_padded: FloatTensor of shape (N, max_size, D) where max_size is max + of `sizes`. The values for batch element i which start at + `inputs[first_idxs[i]]` will be copied to `inputs_padded[i, :]`, + with zeros padding out the extra inputs. + """ + if not (inputs.dim() == 2): + raise ValueError("input can only be 2-dimensional.") + if not (first_idxs.dim() == 1): + raise ValueError("first_idxs can only be 1-dimensional.") + if not (inputs.dtype == torch.float32): + raise ValueError("input has to be of type torch.float32.") + if not (first_idxs.dtype == torch.int64): + raise ValueError("first_idxs has to be of type torch.int64.") + if not isinstance(max_size, int): + raise ValueError("max_size has to be int.") + + ctx.save_for_backward(first_idxs) + ctx.num_inputs = int(inputs.shape[0]) + inputs, first_idxs = inputs.contiguous(), first_idxs.contiguous() + inputs_padded = _C.packed_to_padded(inputs, first_idxs, max_size) + return inputs_padded + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + grad_output = grad_output.contiguous() + first_idxs = ctx.saved_tensors[0] + num_inputs = ctx.num_inputs + grad_input = _C.padded_to_packed(grad_output, first_idxs, num_inputs) + return grad_input, None, None + + +def packed_to_padded( + inputs: torch.Tensor, first_idxs: torch.LongTensor, max_size: int +) -> torch.Tensor: + """ + Torch wrapper that handles allowed input shapes. See description below. + + Args: + inputs: FloatTensor of shape (F,) or (F, ...), representing the packed + batch tensor, e.g. areas for faces in a batch of meshes. + first_idxs: LongTensor of shape (N,) where N is the number of + elements in the batch and `first_idxs[i] = f` + means that the inputs for batch element i begin at `inputs[f]`. + max_size: Max length of an element in the batch. + + Returns: + inputs_padded: FloatTensor of shape (N, max_size) or (N, max_size, ...) + where max_size is max of `sizes`. The values for batch element i + which start at `inputs[first_idxs[i]]` will be copied to + `inputs_padded[i, :]`, with zeros padding out the extra inputs. + + To handle the allowed input shapes, we convert the inputs tensor of shape + (F,) to (F, 1). We reshape the output back to (N, max_size) from + (N, max_size, 1). + """ + # if inputs is of shape (F,), reshape into (F, 1) + input_shape = inputs.shape + n_dims = inputs.dim() + if n_dims == 1: + inputs = inputs.unsqueeze(1) + else: + inputs = inputs.reshape(input_shape[0], -1) + inputs_padded = _PackedToPadded.apply(inputs, first_idxs, max_size) + # if flat is True, reshape output to (N, max_size) from (N, max_size, 1) + # else reshape output to (N, max_size, ...) + if n_dims == 1: + return inputs_padded.squeeze(2) + if n_dims == 2: + return inputs_padded + return inputs_padded.view(*inputs_padded.shape[:2], *input_shape[1:]) + + +class _PaddedToPacked(Function): + """ + Torch autograd Function wrapper for padded_to_packed C++/CUDA implementations. + """ + + @staticmethod + def forward(ctx, inputs, first_idxs, num_inputs): + """ + Args: + ctx: Context object used to calculate gradients. + inputs: FloatTensor of shape (N, max_size, D), representing + the padded tensor, e.g. areas for faces in a batch of meshes. + first_idxs: LongTensor of shape (N,) where N is the number of + elements in the batch and `first_idxs[i] = f` + means that the inputs for batch element i begin at `inputs_packed[f]`. + num_inputs: Number of packed entries (= F) + + Returns: + inputs_packed: FloatTensor of shape (F, D) where + `inputs_packed[first_idx[i]:] = inputs[i, :]`. + """ + if not (inputs.dim() == 3): + raise ValueError("input can only be 3-dimensional.") + if not (first_idxs.dim() == 1): + raise ValueError("first_idxs can only be 1-dimensional.") + if not (inputs.dtype == torch.float32): + raise ValueError("input has to be of type torch.float32.") + if not (first_idxs.dtype == torch.int64): + raise ValueError("first_idxs has to be of type torch.int64.") + if not isinstance(num_inputs, int): + raise ValueError("max_size has to be int.") + + ctx.save_for_backward(first_idxs) + ctx.max_size = inputs.shape[1] + inputs, first_idxs = inputs.contiguous(), first_idxs.contiguous() + inputs_packed = _C.padded_to_packed(inputs, first_idxs, num_inputs) + return inputs_packed + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + grad_output = grad_output.contiguous() + first_idxs = ctx.saved_tensors[0] + max_size = ctx.max_size + grad_input = _C.packed_to_padded(grad_output, first_idxs, max_size) + return grad_input, None, None + + +def padded_to_packed( + inputs: torch.Tensor, + first_idxs: torch.LongTensor, + num_inputs: int, + max_size_dim: int = 1, +) -> torch.Tensor: + """ + Torch wrapper that handles allowed input shapes. See description below. + + Args: + inputs: FloatTensor of shape (N, ..., max_size) or (N, ..., max_size, ...), + representing the padded tensor, e.g. areas for faces in a batch of + meshes, where max_size occurs on max_size_dim-th position. + first_idxs: LongTensor of shape (N,) where N is the number of + elements in the batch and `first_idxs[i] = f` + means that the inputs for batch element i begin at `inputs_packed[f]`. + num_inputs: Number of packed entries (= F) + max_size_dim: the dimension to be packed + + Returns: + inputs_packed: FloatTensor of shape (F,) or (F, ...) where + `inputs_packed[first_idx[i]:first_idx[i+1]] = inputs[i, ..., :delta[i]]`, + where `delta[i] = first_idx[i+1] - first_idx[i]`. + + To handle the allowed input shapes, we convert the inputs tensor of shape + (N, max_size) to (N, max_size, 1). We reshape the output back to (F,) from + (F, 1). + """ + n_dims = inputs.dim() + # move the variable dim to position 1 + inputs = inputs.movedim(max_size_dim, 1) + + # if inputs is of shape (N, max_size), reshape into (N, max_size, 1)) + input_shape = inputs.shape + if n_dims == 2: + inputs = inputs.unsqueeze(2) + else: + inputs = inputs.reshape(*input_shape[:2], -1) + inputs_packed = _PaddedToPacked.apply(inputs, first_idxs, num_inputs) + # if input is flat, reshape output to (F,) from (F, 1) + # else reshape output to (F, ...) + if n_dims == 2: + return inputs_packed.squeeze(1) + + return inputs_packed.view(-1, *input_shape[2:]) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/perspective_n_points.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/perspective_n_points.py new file mode 100644 index 0000000000000000000000000000000000000000..db5e0057bceb6cf7b18b8916a81cad6424701056 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/perspective_n_points.py @@ -0,0 +1,412 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +""" +This file contains Efficient PnP algorithm for Perspective-n-Points problem. +It finds a camera position (defined by rotation `R` and translation `T`) that +minimizes re-projection error between the given 3D points `x` and +the corresponding uncalibrated 2D points `y`. +""" + +import warnings +from typing import NamedTuple, Optional + +import torch +import torch.nn.functional as F +from pytorch3d.ops import points_alignment, utils as oputil + + +class EpnpSolution(NamedTuple): + x_cam: torch.Tensor + R: torch.Tensor + T: torch.Tensor + err_2d: torch.Tensor + err_3d: torch.Tensor + + +def _define_control_points(x, weight, storage_opts=None): + """ + Returns control points that define barycentric coordinates + Args: + x: Batch of 3-dimensional points of shape `(minibatch, num_points, 3)`. + weight: Batch of non-negative weights of + shape `(minibatch, num_point)`. `None` means equal weights. + storage_opts: dict of keyword arguments to the tensor constructor. + """ + storage_opts = storage_opts or {} + x_mean = oputil.wmean(x, weight) + c_world = F.pad(torch.eye(3, **storage_opts), (0, 0, 0, 1), value=0.0).expand_as( + x[:, :4, :] + ) + return c_world + x_mean + + +def _compute_alphas(x, c_world): + """ + Computes barycentric coordinates of x in the frame c_world. + Args: + x: Batch of 3-dimensional points of shape `(minibatch, num_points, 3)`. + c_world: control points in world coordinates. + """ + x = F.pad(x, (0, 1), value=1.0) + c = F.pad(c_world, (0, 1), value=1.0) + return torch.matmul(x, torch.inverse(c)) # B x N x 4 + + +def _build_M(y, alphas, weight): + """Returns the matrix defining the reprojection equations. + Args: + y: projected points in camera coordinates of size B x N x 2 + alphas: barycentric coordinates of size B x N x 4 + weight: Batch of non-negative weights of + shape `(minibatch, num_point)`. `None` means equal weights. + """ + bs, n, _ = y.size() + + # prepend t with the column of v's + def prepad(t, v): + return F.pad(t, (1, 0), value=v) + + if weight is not None: + # weight the alphas in order to get a correctly weighted version of M + alphas = alphas * weight[:, :, None] + + # outer left-multiply by alphas + def lm_alphas(t): + return torch.matmul(alphas[..., None], t).reshape(bs, n, 12) + + M = torch.cat( + ( + lm_alphas( + prepad(prepad(-y[:, :, 0, None, None], 0.0), 1.0) + ), # u constraints + lm_alphas( + prepad(prepad(-y[:, :, 1, None, None], 1.0), 0.0) + ), # v constraints + ), + dim=-1, + ).reshape(bs, -1, 12) + + return M + + +def _null_space(m, kernel_dim): + """Finds the null space (kernel) basis of the matrix + Args: + m: the batch of input matrices, B x N x 12 + kernel_dim: number of dimensions to approximate the kernel + Returns: + * a batch of null space basis vectors + of size B x 4 x 3 x kernel_dim + * a batch of spectral values where near-0s correspond to actual + kernel vectors, of size B x kernel_dim + """ + mTm = torch.bmm(m.transpose(1, 2), m) + s, v = torch.linalg.eigh(mTm) + return v[:, :, :kernel_dim].reshape(-1, 4, 3, kernel_dim), s[:, :kernel_dim] + + +def _reproj_error(y_hat, y, weight, eps=1e-9): + """Projects estimated 3D points and computes the reprojection error + Args: + y_hat: a batch of predicted 2D points in homogeneous coordinates + y: a batch of ground-truth 2D points + weight: Batch of non-negative weights of + shape `(minibatch, num_point)`. `None` means equal weights. + Returns: + Optionally weighted RMSE of difference between y and y_hat. + """ + y_hat = y_hat / torch.clamp(y_hat[..., 2:], eps) + dist = ((y - y_hat[..., :2]) ** 2).sum(dim=-1, keepdim=True) ** 0.5 + return oputil.wmean(dist, weight)[:, 0, 0] + + +def _algebraic_error(x_w_rotated, x_cam, weight): + """Computes the residual of Umeyama in 3D. + Args: + x_w_rotated: The given 3D points rotated with the predicted camera. + x_cam: the lifted 2D points y + weight: Batch of non-negative weights of + shape `(minibatch, num_point)`. `None` means equal weights. + Returns: + Optionally weighted MSE of difference between x_w_rotated and x_cam. + """ + dist = ((x_w_rotated - x_cam) ** 2).sum(dim=-1, keepdim=True) + return oputil.wmean(dist, weight)[:, 0, 0] + + +def _compute_norm_sign_scaling_factor(c_cam, alphas, x_world, y, weight, eps=1e-9): + """Given a solution, adjusts the scale and flip + Args: + c_cam: control points in camera coordinates + alphas: barycentric coordinates of the points + x_world: Batch of 3-dimensional points of shape `(minibatch, num_points, 3)`. + y: Batch of 2-dimensional points of shape `(minibatch, num_points, 2)`. + weights: Batch of non-negative weights of + shape `(minibatch, num_point)`. `None` means equal weights. + eps: epsilon to threshold negative `z` values + """ + # position of reference points in camera coordinates + x_cam = torch.matmul(alphas, c_cam) + + x_cam = x_cam * (1.0 - 2.0 * (oputil.wmean(x_cam[..., 2:], weight) < 0).float()) + if torch.any(x_cam[..., 2:] < -eps): + neg_rate = oputil.wmean((x_cam[..., 2:] < 0).float(), weight, dim=(0, 1)).item() + warnings.warn("\nEPnP: %2.2f%% points have z<0." % (neg_rate * 100.0)) + + R, T, s = points_alignment.corresponding_points_alignment( + x_world, x_cam, weight, estimate_scale=True + ) + s = s.clamp(eps) + x_cam = x_cam / s[:, None, None] + T = T / s[:, None] + x_w_rotated = torch.matmul(x_world, R) + T[:, None, :] + err_2d = _reproj_error(x_w_rotated, y, weight) + err_3d = _algebraic_error(x_w_rotated, x_cam, weight) + + return EpnpSolution(x_cam, R, T, err_2d, err_3d) + + +def _gen_pairs(input, dim=-2, reducer=lambda a, b: ((a - b) ** 2).sum(dim=-1)): + """Generates all pairs of different rows and then applies the reducer + Args: + input: a tensor + dim: a dimension to generate pairs across + reducer: a function of generated pair of rows to apply (beyond just concat) + Returns: + for default args, for A x B x C input, will output A x (B choose 2) + """ + n = input.size()[dim] + range = torch.arange(n) + idx = torch.combinations(range).to(input).long() + left = input.index_select(dim, idx[:, 0]) + right = input.index_select(dim, idx[:, 1]) + return reducer(left, right) + + +def _kernel_vec_distances(v): + """Computes the coefficients for linearization of the quadratic system + to match all pairwise distances between 4 control points (dim=1). + The last dimension corresponds to the coefficients for quadratic terms + Bij = Bi * Bj, where Bi and Bj correspond to kernel vectors. + Arg: + v: tensor of B x 4 x 3 x D, where D is dim(kernel), usually 4 + Returns: + a tensor of B x 6 x [(D choose 2) + D]; + for D=4, the last dim means [B11 B22 B33 B44 B12 B13 B14 B23 B24 B34]. + """ + dv = _gen_pairs(v, dim=-3, reducer=lambda a, b: a - b) # B x 6 x 3 x D + + # we should take dot-product of all (i,j), i < j, with coeff 2 + rows_2ij = 2.0 * _gen_pairs(dv, dim=-1, reducer=lambda a, b: (a * b).sum(dim=-2)) + # this should produce B x 6 x (D choose 2) tensor + + # we should take dot-product of all (i,i) + rows_ii = (dv**2).sum(dim=-2) + # this should produce B x 6 x D tensor + + return torch.cat((rows_ii, rows_2ij), dim=-1) + + +def _solve_lstsq_subcols(rhs, lhs, lhs_col_idx): + """Solves an over-determined linear system for selected LHS columns. + A batched version of `torch.lstsq`. + Args: + rhs: right-hand side vectors + lhs: left-hand side matrices + lhs_col_idx: a slice of columns in lhs + Returns: + a least-squares solution for lhs * X = rhs + """ + lhs = lhs.index_select(-1, torch.tensor(lhs_col_idx, device=lhs.device).long()) + return torch.matmul(torch.pinverse(lhs), rhs[:, :, None]) + + +def _binary_sign(t): + return (t >= 0).to(t) * 2.0 - 1.0 + + +def _find_null_space_coords_1(kernel_dsts, cw_dst, eps=1e-9): + """Solves case 1 from the paper [1]; solve for 4 coefficients: + [B11 B22 B33 B44 B12 B13 B14 B23 B24 B34] + ^ ^ ^ ^ + Args: + kernel_dsts: distances between kernel vectors + cw_dst: distances between control points + Returns: + coefficients to weight kernel vectors + [1] Moreno-Noguer, F., Lepetit, V., & Fua, P. (2009). + EPnP: An Accurate O(n) solution to the PnP problem. + International Journal of Computer Vision. + https://www.epfl.ch/labs/cvlab/software/multi-view-stereo/epnp/ + """ + beta = _solve_lstsq_subcols(cw_dst, kernel_dsts, [0, 4, 5, 6]) + + beta = beta * _binary_sign(beta[:, :1, :]) + return beta / torch.clamp(beta[:, :1, :] ** 0.5, eps) + + +def _find_null_space_coords_2(kernel_dsts, cw_dst): + """Solves case 2 from the paper; solve for 3 coefficients: + [B11 B22 B33 B44 B12 B13 B14 B23 B24 B34] + ^ ^ ^ + Args: + kernel_dsts: distances between kernel vectors + cw_dst: distances between control points + Returns: + coefficients to weight kernel vectors + [1] Moreno-Noguer, F., Lepetit, V., & Fua, P. (2009). + EPnP: An Accurate O(n) solution to the PnP problem. + International Journal of Computer Vision. + https://www.epfl.ch/labs/cvlab/software/multi-view-stereo/epnp/ + """ + beta = _solve_lstsq_subcols(cw_dst, kernel_dsts, [0, 4, 1]) + + coord_0 = (beta[:, :1, :].abs() ** 0.5) * _binary_sign(beta[:, 1:2, :]) + coord_1 = (beta[:, 2:3, :].abs() ** 0.5) * ( + (beta[:, :1, :] >= 0) == (beta[:, 2:3, :] >= 0) + ).float() + + return torch.cat((coord_0, coord_1, torch.zeros_like(beta[:, :2, :])), dim=1) + + +def _find_null_space_coords_3(kernel_dsts, cw_dst, eps=1e-9): + """Solves case 3 from the paper; solve for 5 coefficients: + [B11 B22 B33 B44 B12 B13 B14 B23 B24 B34] + ^ ^ ^ ^ ^ + Args: + kernel_dsts: distances between kernel vectors + cw_dst: distances between control points + Returns: + coefficients to weight kernel vectors + [1] Moreno-Noguer, F., Lepetit, V., & Fua, P. (2009). + EPnP: An Accurate O(n) solution to the PnP problem. + International Journal of Computer Vision. + https://www.epfl.ch/labs/cvlab/software/multi-view-stereo/epnp/ + """ + beta = _solve_lstsq_subcols(cw_dst, kernel_dsts, [0, 4, 1, 5, 7]) + + coord_0 = (beta[:, :1, :].abs() ** 0.5) * _binary_sign(beta[:, 1:2, :]) + coord_1 = (beta[:, 2:3, :].abs() ** 0.5) * ( + (beta[:, :1, :] >= 0) == (beta[:, 2:3, :] >= 0) + ).float() + coord_2 = beta[:, 3:4, :] / torch.clamp(coord_0[:, :1, :], eps) + + return torch.cat( + (coord_0, coord_1, coord_2, torch.zeros_like(beta[:, :1, :])), dim=1 + ) + + +def efficient_pnp( + x: torch.Tensor, + y: torch.Tensor, + weights: Optional[torch.Tensor] = None, + skip_quadratic_eq: bool = False, +) -> EpnpSolution: + """ + Implements Efficient PnP algorithm [1] for Perspective-n-Points problem: + finds a camera position (defined by rotation `R` and translation `T`) that + minimizes re-projection error between the given 3D points `x` and + the corresponding uncalibrated 2D points `y`, i.e. solves + + `y[i] = Proj(x[i] R[i] + T[i])` + + in the least-squares sense, where `i` are indices within the batch, and + `Proj` is the perspective projection operator: `Proj([x y z]) = [x/z y/z]`. + In the noise-less case, 4 points are enough to find the solution as long + as they are not co-planar. + + Args: + x: Batch of 3-dimensional points of shape `(minibatch, num_points, 3)`. + y: Batch of 2-dimensional points of shape `(minibatch, num_points, 2)`. + weights: Batch of non-negative weights of + shape `(minibatch, num_point)`. `None` means equal weights. + skip_quadratic_eq: If True, assumes the solution space for the + linear system is one-dimensional, i.e. takes the scaled eigenvector + that corresponds to the smallest eigenvalue as a solution. + If False, finds the candidate coordinates in the potentially + 4D null space by approximately solving the systems of quadratic + equations. The best candidate is chosen by examining the 2D + re-projection error. While this option finds a better solution, + especially when the number of points is small or perspective + distortions are low (the points are far away), it may be more + difficult to back-propagate through. + + Returns: + `EpnpSolution` namedtuple containing elements: + **x_cam**: Batch of transformed points `x` that is used to find + the camera parameters, of shape `(minibatch, num_points, 3)`. + In the general (noisy) case, they are not exactly equal to + `x[i] R[i] + T[i]` but are some affine transform of `x[i]`s. + **R**: Batch of rotation matrices of shape `(minibatch, 3, 3)`. + **T**: Batch of translation vectors of shape `(minibatch, 3)`. + **err_2d**: Batch of mean 2D re-projection errors of shape + `(minibatch,)`. Specifically, if `yhat` is the re-projection for + the `i`-th batch element, it returns `sum_j norm(yhat_j - y_j)` + where `j` iterates over points and `norm` denotes the L2 norm. + **err_3d**: Batch of mean algebraic errors of shape `(minibatch,)`. + Specifically, those are squared distances between `x_world` and + estimated points on the rays defined by `y`. + + [1] Moreno-Noguer, F., Lepetit, V., & Fua, P. (2009). + EPnP: An Accurate O(n) solution to the PnP problem. + International Journal of Computer Vision. + https://www.epfl.ch/labs/cvlab/software/multi-view-stereo/epnp/ + """ + # define control points in a world coordinate system (centered on the 3d + # points centroid); 4 x 3 + # TODO: more stable when initialised with the center and eigenvectors! + c_world = _define_control_points( + x.detach(), weights, storage_opts={"dtype": x.dtype, "device": x.device} + ) + + # find the linear combination of the control points to represent the 3d points + alphas = _compute_alphas(x, c_world) + + M = _build_M(y, alphas, weights) + + # Compute kernel M + kernel, spectrum = _null_space(M, 4) + + c_world_distances = _gen_pairs(c_world) + kernel_dsts = _kernel_vec_distances(kernel) + + betas = ( + [] + if skip_quadratic_eq + else [ + fnsc(kernel_dsts, c_world_distances) + for fnsc in [ + _find_null_space_coords_1, + _find_null_space_coords_2, + _find_null_space_coords_3, + ] + ] + ) + + c_cam_variants = [kernel] + [ + torch.matmul(kernel, beta[:, None, :, :]) for beta in betas + ] + + solutions = [ + _compute_norm_sign_scaling_factor(c_cam[..., 0], alphas, x, y, weights) + for c_cam in c_cam_variants + ] + + sol_zipped = EpnpSolution(*(torch.stack(list(col)) for col in zip(*solutions))) + best = torch.argmin(sol_zipped.err_2d, dim=0) + + def gather1d(source, idx): + # reduces the dim=1 by picking the slices in a 1D tensor idx + # in other words, it is batched index_select. + return source.gather( + 0, + idx.reshape(1, -1, *([1] * (len(source.shape) - 2))).expand_as(source[:1]), + )[0] + + return EpnpSolution(*[gather1d(sol_col, best) for sol_col in sol_zipped]) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/points_alignment.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/points_alignment.py new file mode 100644 index 0000000000000000000000000000000000000000..96e4b410a772d7a26df663580d4f47974b8cbdbb --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/points_alignment.py @@ -0,0 +1,391 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import warnings +from typing import List, NamedTuple, Optional, TYPE_CHECKING, Union + +import torch +from pytorch3d.ops import knn_points +from pytorch3d.structures import utils as strutil + +from . import utils as oputil + + +if TYPE_CHECKING: + from pytorch3d.structures.pointclouds import Pointclouds + + +# named tuples for inputs/outputs +class SimilarityTransform(NamedTuple): + R: torch.Tensor + T: torch.Tensor + s: torch.Tensor + + +class ICPSolution(NamedTuple): + converged: bool + rmse: Union[torch.Tensor, None] + Xt: torch.Tensor + RTs: SimilarityTransform + t_history: List[SimilarityTransform] + + +def iterative_closest_point( + X: Union[torch.Tensor, "Pointclouds"], + Y: Union[torch.Tensor, "Pointclouds"], + init_transform: Optional[SimilarityTransform] = None, + max_iterations: int = 100, + relative_rmse_thr: float = 1e-6, + estimate_scale: bool = False, + allow_reflection: bool = False, + verbose: bool = False, +) -> ICPSolution: + """ + Executes the iterative closest point (ICP) algorithm [1, 2] in order to find + a similarity transformation (rotation `R`, translation `T`, and + optionally scale `s`) between two given differently-sized sets of + `d`-dimensional points `X` and `Y`, such that: + + `s[i] X[i] R[i] + T[i] = Y[NN[i]]`, + + for all batch indices `i` in the least squares sense. Here, Y[NN[i]] stands + for the indices of nearest neighbors from `Y` to each point in `X`. + Note, however, that the solution is only a local optimum. + + Args: + **X**: Batch of `d`-dimensional points + of shape `(minibatch, num_points_X, d)` or a `Pointclouds` object. + **Y**: Batch of `d`-dimensional points + of shape `(minibatch, num_points_Y, d)` or a `Pointclouds` object. + **init_transform**: A named-tuple `SimilarityTransform` of tensors + `R`, `T, `s`, where `R` is a batch of orthonormal matrices of + shape `(minibatch, d, d)`, `T` is a batch of translations + of shape `(minibatch, d)` and `s` is a batch of scaling factors + of shape `(minibatch,)`. + **max_iterations**: The maximum number of ICP iterations. + **relative_rmse_thr**: A threshold on the relative root mean squared error + used to terminate the algorithm. + **estimate_scale**: If `True`, also estimates a scaling component `s` + of the transformation. Otherwise assumes the identity + scale and returns a tensor of ones. + **allow_reflection**: If `True`, allows the algorithm to return `R` + which is orthonormal but has determinant==-1. + **verbose**: If `True`, prints status messages during each ICP iteration. + + Returns: + A named tuple `ICPSolution` with the following fields: + **converged**: A boolean flag denoting whether the algorithm converged + successfully (=`True`) or not (=`False`). + **rmse**: Attained root mean squared error after termination of ICP. + **Xt**: The point cloud `X` transformed with the final transformation + (`R`, `T`, `s`). If `X` is a `Pointclouds` object, returns an + instance of `Pointclouds`, otherwise returns `torch.Tensor`. + **RTs**: A named tuple `SimilarityTransform` containing + a batch of similarity transforms with fields: + **R**: Batch of orthonormal matrices of shape `(minibatch, d, d)`. + **T**: Batch of translations of shape `(minibatch, d)`. + **s**: batch of scaling factors of shape `(minibatch, )`. + **t_history**: A list of named tuples `SimilarityTransform` + the transformation parameters after each ICP iteration. + + References: + [1] Besl & McKay: A Method for Registration of 3-D Shapes. TPAMI, 1992. + [2] https://en.wikipedia.org/wiki/Iterative_closest_point + """ + + # make sure we convert input Pointclouds structures to + # padded tensors of shape (N, P, 3) + Xt, num_points_X = oputil.convert_pointclouds_to_tensor(X) + Yt, num_points_Y = oputil.convert_pointclouds_to_tensor(Y) + + b, size_X, dim = Xt.shape + + if (Xt.shape[2] != Yt.shape[2]) or (Xt.shape[0] != Yt.shape[0]): + raise ValueError( + "Point sets X and Y have to have the same " + + "number of batches and data dimensions." + ) + + if ((num_points_Y < Yt.shape[1]).any() or (num_points_X < Xt.shape[1]).any()) and ( + num_points_Y != num_points_X + ).any(): + # we have a heterogeneous input (e.g. because X/Y is + # an instance of Pointclouds) + mask_X = ( + torch.arange(size_X, dtype=torch.int64, device=Xt.device)[None] + < num_points_X[:, None] + ).type_as(Xt) + else: + mask_X = Xt.new_ones(b, size_X) + + # clone the initial point cloud + Xt_init = Xt.clone() + + if init_transform is not None: + # parse the initial transform from the input and apply to Xt + try: + R, T, s = init_transform + assert ( + R.shape == torch.Size((b, dim, dim)) + and T.shape == torch.Size((b, dim)) + and s.shape == torch.Size((b,)) + ) + except Exception: + raise ValueError( + "The initial transformation init_transform has to be " + "a named tuple SimilarityTransform with elements (R, T, s). " + "R are dim x dim orthonormal matrices of shape " + "(minibatch, dim, dim), T is a batch of dim-dimensional " + "translations of shape (minibatch, dim) and s is a batch " + "of scalars of shape (minibatch,)." + ) from None + # apply the init transform to the input point cloud + Xt = _apply_similarity_transform(Xt, R, T, s) + else: + # initialize the transformation with identity + R = oputil.eyes(dim, b, device=Xt.device, dtype=Xt.dtype) + T = Xt.new_zeros((b, dim)) + s = Xt.new_ones(b) + + prev_rmse = None + rmse = None + iteration = -1 + converged = False + + # initialize the transformation history + t_history = [] + + # the main loop over ICP iterations + for iteration in range(max_iterations): + Xt_nn_points = knn_points( + Xt, Yt, lengths1=num_points_X, lengths2=num_points_Y, K=1, return_nn=True + ).knn[:, :, 0, :] + + # get the alignment of the nearest neighbors from Yt with Xt_init + R, T, s = corresponding_points_alignment( + Xt_init, + Xt_nn_points, + weights=mask_X, + estimate_scale=estimate_scale, + allow_reflection=allow_reflection, + ) + + # apply the estimated similarity transform to Xt_init + Xt = _apply_similarity_transform(Xt_init, R, T, s) + + # add the current transformation to the history + t_history.append(SimilarityTransform(R, T, s)) + + # compute the root mean squared error + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and `int`. + Xt_sq_diff = ((Xt - Xt_nn_points) ** 2).sum(2) + rmse = oputil.wmean(Xt_sq_diff[:, :, None], mask_X).sqrt()[:, 0, 0] + + # compute the relative rmse + if prev_rmse is None: + relative_rmse = rmse.new_ones(b) + else: + relative_rmse = (prev_rmse - rmse) / prev_rmse + + if verbose: + rmse_msg = ( + f"ICP iteration {iteration}: mean/max rmse = " + + f"{rmse.mean():1.2e}/{rmse.max():1.2e} " + + f"; mean relative rmse = {relative_rmse.mean():1.2e}" + ) + print(rmse_msg) + + # check for convergence + if (relative_rmse <= relative_rmse_thr).all(): + converged = True + break + + # update the previous rmse + prev_rmse = rmse + + if verbose: + if converged: + print(f"ICP has converged in {iteration + 1} iterations.") + else: + print(f"ICP has not converged in {max_iterations} iterations.") + + if oputil.is_pointclouds(X): + Xt = X.update_padded(Xt) # type: ignore + + return ICPSolution(converged, rmse, Xt, SimilarityTransform(R, T, s), t_history) + + +# threshold for checking that point crosscorelation +# is full rank in corresponding_points_alignment +AMBIGUOUS_ROT_SINGULAR_THR = 1e-15 + + +def corresponding_points_alignment( + X: Union[torch.Tensor, "Pointclouds"], + Y: Union[torch.Tensor, "Pointclouds"], + weights: Union[torch.Tensor, List[torch.Tensor], None] = None, + estimate_scale: bool = False, + allow_reflection: bool = False, + eps: float = 1e-9, +) -> SimilarityTransform: + """ + Finds a similarity transformation (rotation `R`, translation `T` + and optionally scale `s`) between two given sets of corresponding + `d`-dimensional points `X` and `Y` such that: + + `s[i] X[i] R[i] + T[i] = Y[i]`, + + for all batch indexes `i` in the least squares sense. + + The algorithm is also known as Umeyama [1]. + + Args: + **X**: Batch of `d`-dimensional points of shape `(minibatch, num_point, d)` + or a `Pointclouds` object. + **Y**: Batch of `d`-dimensional points of shape `(minibatch, num_point, d)` + or a `Pointclouds` object. + **weights**: Batch of non-negative weights of + shape `(minibatch, num_point)` or list of `minibatch` 1-dimensional + tensors that may have different shapes; in that case, the length of + i-th tensor should be equal to the number of points in X_i and Y_i. + Passing `None` means uniform weights. + **estimate_scale**: If `True`, also estimates a scaling component `s` + of the transformation. Otherwise assumes an identity + scale and returns a tensor of ones. + **allow_reflection**: If `True`, allows the algorithm to return `R` + which is orthonormal but has determinant==-1. + **eps**: A scalar for clamping to avoid dividing by zero. Active for the + code that estimates the output scale `s`. + + Returns: + 3-element named tuple `SimilarityTransform` containing + - **R**: Batch of orthonormal matrices of shape `(minibatch, d, d)`. + - **T**: Batch of translations of shape `(minibatch, d)`. + - **s**: batch of scaling factors of shape `(minibatch, )`. + + References: + [1] Shinji Umeyama: Least-Suqares Estimation of + Transformation Parameters Between Two Point Patterns + """ + + # make sure we convert input Pointclouds structures to tensors + Xt, num_points = oputil.convert_pointclouds_to_tensor(X) + Yt, num_points_Y = oputil.convert_pointclouds_to_tensor(Y) + + if (Xt.shape != Yt.shape) or (num_points != num_points_Y).any(): + raise ValueError( + "Point sets X and Y have to have the same \ + number of batches, points and dimensions." + ) + if weights is not None: + if isinstance(weights, list): + if any(np != w.shape[0] for np, w in zip(num_points, weights)): + raise ValueError( + "number of weights should equal to the " + + "number of points in the point cloud." + ) + weights = [w[..., None] for w in weights] + weights = strutil.list_to_padded(weights)[..., 0] + + if Xt.shape[:2] != weights.shape: + raise ValueError("weights should have the same first two dimensions as X.") + + b, n, dim = Xt.shape + + if (num_points < Xt.shape[1]).any() or (num_points < Yt.shape[1]).any(): + # in case we got Pointclouds as input, mask the unused entries in Xc, Yc + mask = ( + torch.arange(n, dtype=torch.int64, device=Xt.device)[None] + < num_points[:, None] + ).type_as(Xt) + weights = mask if weights is None else mask * weights.type_as(Xt) + + # compute the centroids of the point sets + Xmu = oputil.wmean(Xt, weight=weights, eps=eps) + Ymu = oputil.wmean(Yt, weight=weights, eps=eps) + + # mean-center the point sets + Xc = Xt - Xmu + Yc = Yt - Ymu + + total_weight = torch.clamp(num_points, 1) + # special handling for heterogeneous point clouds and/or input weights + if weights is not None: + Xc *= weights[:, :, None] + Yc *= weights[:, :, None] + total_weight = torch.clamp(weights.sum(1), eps) + + if (num_points < (dim + 1)).any(): + warnings.warn( + "The size of one of the point clouds is <= dim+1. " + + "corresponding_points_alignment cannot return a unique rotation." + ) + + # compute the covariance XYcov between the point sets Xc, Yc + XYcov = torch.bmm(Xc.transpose(2, 1), Yc) + XYcov = XYcov / total_weight[:, None, None] + + # decompose the covariance matrix XYcov + U, S, V = torch.svd(XYcov) + + # catch ambiguous rotation by checking the magnitude of singular values + if (S.abs() <= AMBIGUOUS_ROT_SINGULAR_THR).any() and not ( + num_points < (dim + 1) + ).any(): + warnings.warn( + "Excessively low rank of " + + "cross-correlation between aligned point clouds. " + + "corresponding_points_alignment cannot return a unique rotation." + ) + + # identity matrix used for fixing reflections + E = torch.eye(dim, dtype=XYcov.dtype, device=XYcov.device)[None].repeat(b, 1, 1) + + if not allow_reflection: + # reflection test: + # checks whether the estimated rotation has det==1, + # if not, finds the nearest rotation s.t. det==1 by + # flipping the sign of the last singular vector U + R_test = torch.bmm(U, V.transpose(2, 1)) + E[:, -1, -1] = torch.det(R_test) + + # find the rotation matrix by composing U and V again + R = torch.bmm(torch.bmm(U, E), V.transpose(2, 1)) + + if estimate_scale: + # estimate the scaling component of the transformation + trace_ES = (torch.diagonal(E, dim1=1, dim2=2) * S).sum(1) + Xcov = (Xc * Xc).sum((1, 2)) / total_weight + + # the scaling component + s = trace_ES / torch.clamp(Xcov, eps) + + # translation component + T = Ymu[:, 0, :] - s[:, None] * torch.bmm(Xmu, R)[:, 0, :] + else: + # translation component + T = Ymu[:, 0, :] - torch.bmm(Xmu, R)[:, 0, :] + + # unit scaling since we do not estimate scale + s = T.new_ones(b) + + return SimilarityTransform(R, T, s) + + +def _apply_similarity_transform( + X: torch.Tensor, R: torch.Tensor, T: torch.Tensor, s: torch.Tensor +) -> torch.Tensor: + """ + Applies a similarity transformation parametrized with a batch of orthonormal + matrices `R` of shape `(minibatch, d, d)`, a batch of translations `T` + of shape `(minibatch, d)` and a batch of scaling factors `s` + of shape `(minibatch,)` to a given `d`-dimensional cloud `X` + of shape `(minibatch, num_points, d)` + """ + X = s[:, None, None] * torch.bmm(X, R) + T[:, None, :] + return X diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/points_normals.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/points_normals.py new file mode 100644 index 0000000000000000000000000000000000000000..f4d3b27ce1284e1af38b64596366019a5b4cc962 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/points_normals.py @@ -0,0 +1,191 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from typing import Tuple, TYPE_CHECKING, Union + +import torch +from pytorch3d.common.workaround import symeig3x3 + +from .utils import convert_pointclouds_to_tensor, get_point_covariances + + +if TYPE_CHECKING: + from ..structures import Pointclouds + + +def estimate_pointcloud_normals( + pointclouds: Union[torch.Tensor, "Pointclouds"], + neighborhood_size: int = 50, + disambiguate_directions: bool = True, + *, + use_symeig_workaround: bool = True, +) -> torch.Tensor: + """ + Estimates the normals of a batch of `pointclouds`. + + The function uses `estimate_pointcloud_local_coord_frames` to estimate + the normals. Please refer to that function for more detailed information. + + Args: + **pointclouds**: Batch of 3-dimensional points of shape + `(minibatch, num_point, 3)` or a `Pointclouds` object. + **neighborhood_size**: The size of the neighborhood used to estimate the + geometry around each point. + **disambiguate_directions**: If `True`, uses the algorithm from [1] to + ensure sign consistency of the normals of neighboring points. + **use_symeig_workaround**: If `True`, uses a custom eigenvalue + calculation. + + Returns: + **normals**: A tensor of normals for each input point + of shape `(minibatch, num_point, 3)`. + If `pointclouds` are of `Pointclouds` class, returns a padded tensor. + + References: + [1] Tombari, Salti, Di Stefano: Unique Signatures of Histograms for + Local Surface Description, ECCV 2010. + """ + + curvatures, local_coord_frames = estimate_pointcloud_local_coord_frames( + pointclouds, + neighborhood_size=neighborhood_size, + disambiguate_directions=disambiguate_directions, + use_symeig_workaround=use_symeig_workaround, + ) + + # the normals correspond to the first vector of each local coord frame + normals = local_coord_frames[:, :, :, 0] + + return normals + + +def estimate_pointcloud_local_coord_frames( + pointclouds: Union[torch.Tensor, "Pointclouds"], + neighborhood_size: int = 50, + disambiguate_directions: bool = True, + *, + use_symeig_workaround: bool = True, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Estimates the principal directions of curvature (which includes normals) + of a batch of `pointclouds`. + + The algorithm first finds `neighborhood_size` nearest neighbors for each + point of the point clouds, followed by obtaining principal vectors of + covariance matrices of each of the point neighborhoods. + The main principal vector corresponds to the normals, while the + other 2 are the direction of the highest curvature and the 2nd highest + curvature. + + Note that each principal direction is given up to a sign. Hence, + the function implements `disambiguate_directions` switch that allows + to ensure consistency of the sign of neighboring normals. The implementation + follows the sign disabiguation from SHOT descriptors [1]. + + The algorithm also returns the curvature values themselves. + These are the eigenvalues of the estimated covariance matrices + of each point neighborhood. + + Args: + **pointclouds**: Batch of 3-dimensional points of shape + `(minibatch, num_point, 3)` or a `Pointclouds` object. + **neighborhood_size**: The size of the neighborhood used to estimate the + geometry around each point. + **disambiguate_directions**: If `True`, uses the algorithm from [1] to + ensure sign consistency of the normals of neighboring points. + **use_symeig_workaround**: If `True`, uses a custom eigenvalue + calculation. + + Returns: + **curvatures**: The three principal curvatures of each point + of shape `(minibatch, num_point, 3)`. + If `pointclouds` are of `Pointclouds` class, returns a padded tensor. + **local_coord_frames**: The three principal directions of the curvature + around each point of shape `(minibatch, num_point, 3, 3)`. + The principal directions are stored in columns of the output. + E.g. `local_coord_frames[i, j, :, 0]` is the normal of + `j`-th point in the `i`-th pointcloud. + If `pointclouds` are of `Pointclouds` class, returns a padded tensor. + + References: + [1] Tombari, Salti, Di Stefano: Unique Signatures of Histograms for + Local Surface Description, ECCV 2010. + """ + + points_padded, num_points = convert_pointclouds_to_tensor(pointclouds) + + ba, N, dim = points_padded.shape + if dim != 3: + raise ValueError( + "The pointclouds argument has to be of shape (minibatch, N, 3)" + ) + + if (num_points <= neighborhood_size).any(): + raise ValueError( + "The neighborhood_size argument has to be" + + " >= size of each of the point clouds." + ) + + # undo global mean for stability + # TODO: replace with tutil.wmean once landed + pcl_mean = points_padded.sum(1) / num_points[:, None] + points_centered = points_padded - pcl_mean[:, None, :] + + # get the per-point covariance and nearest neighbors used to compute it + cov, knns = get_point_covariances(points_centered, num_points, neighborhood_size) + + # get the local coord frames as principal directions of + # the per-point covariance + # this is done with torch.symeig / torch.linalg.eigh, which returns the + # eigenvectors (=principal directions) in an ascending order of their + # corresponding eigenvalues, and the smallest eigenvalue's eigenvector + # corresponds to the normal direction; or with a custom equivalent. + if use_symeig_workaround: + curvatures, local_coord_frames = symeig3x3(cov, eigenvectors=True) + else: + curvatures, local_coord_frames = torch.linalg.eigh(cov) + + # disambiguate the directions of individual principal vectors + if disambiguate_directions: + # disambiguate normal + n = _disambiguate_vector_directions( + points_centered, knns, local_coord_frames[:, :, :, 0] + ) + # disambiguate the main curvature + z = _disambiguate_vector_directions( + points_centered, knns, local_coord_frames[:, :, :, 2] + ) + # the secondary curvature is just a cross between n and z + y = torch.cross(n, z, dim=2) + # cat to form the set of principal directions + local_coord_frames = torch.stack((n, y, z), dim=3) + + return curvatures, local_coord_frames + + +def _disambiguate_vector_directions(pcl, knns, vecs: torch.Tensor) -> torch.Tensor: + """ + Disambiguates normal directions according to [1]. + + References: + [1] Tombari, Salti, Di Stefano: Unique Signatures of Histograms for + Local Surface Description, ECCV 2010. + """ + # parse out K from the shape of knns + K = knns.shape[2] + # the difference between the mean of each neighborhood and + # each element of the neighborhood + df = knns - pcl[:, :, None] + # projection of the difference on the principal direction + proj = (vecs[:, :, None] * df).sum(3) + # check how many projections are positive + n_pos = (proj > 0).type_as(knns).sum(2, keepdim=True) + # flip the principal directions where number of positive correlations + flip = (n_pos < (0.5 * K)).type_as(knns) + vecs = (1.0 - 2.0 * flip) * vecs + return vecs diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/points_to_volumes.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/points_to_volumes.py new file mode 100644 index 0000000000000000000000000000000000000000..9e051ae394eb96681046674d8f0f7f4f917ccae1 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/points_to_volumes.py @@ -0,0 +1,764 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from typing import Optional, Tuple, TYPE_CHECKING + +import torch +from pytorch3d import _C +from torch.autograd import Function +from torch.autograd.function import once_differentiable + + +if TYPE_CHECKING: + from ..structures import Pointclouds, Volumes + + +class _points_to_volumes_function(Function): + """ + For each point in a pointcloud, add point_weight to the + corresponding volume density and point_weight times its features + to the corresponding volume features. + + This function does not require any contiguity internally and therefore + doesn't need to make copies of its inputs, which is useful when GPU memory + is at a premium. (An implementation requiring contiguous inputs might be faster + though). The volumes are modified in place. + + This function is differentiable with respect to + points_features, volume_densities and volume_features. + If splat is True then it is also differentiable with respect to + points_3d. + + It may be useful to think about this function as a sort of opposite to + torch.nn.functional.grid_sample with 5D inputs. + + Args: + points_3d: Batch of 3D point cloud coordinates of shape + `(minibatch, N, 3)` where N is the number of points + in each point cloud. Coordinates have to be specified in the + local volume coordinates (ranging in [-1, 1]). + points_features: Features of shape `(minibatch, N, feature_dim)` + corresponding to the points of the input point cloud `points_3d`. + volume_features: Batch of input feature volumes + of shape `(minibatch, feature_dim, D, H, W)` + volume_densities: Batch of input feature volume densities + of shape `(minibatch, 1, D, H, W)`. Each voxel should + contain a non-negative number corresponding to its + opaqueness (the higher, the less transparent). + + grid_sizes: `LongTensor` of shape (minibatch, 3) representing the + spatial resolutions of each of the the non-flattened `volumes` + tensors. Note that the following has to hold: + `torch.prod(grid_sizes, dim=1)==N_voxels`. + + point_weight: A scalar controlling how much weight a single point has. + + mask: A binary mask of shape `(minibatch, N)` determining + which 3D points are going to be converted to the resulting + volume. Set to `None` if all points are valid. + + align_corners: as for grid_sample. + + splat: if true, trilinear interpolation. If false all the weight goes in + the nearest voxel. + + Returns: + volume_densities and volume_features, which have been modified in place. + """ + + @staticmethod + # pyre-fixme[14]: `forward` overrides method defined in `Function` inconsistently. + def forward( + ctx, + points_3d: torch.Tensor, + points_features: torch.Tensor, + volume_densities: torch.Tensor, + volume_features: torch.Tensor, + grid_sizes: torch.LongTensor, + point_weight: float, + mask: torch.Tensor, + align_corners: bool, + splat: bool, + ): + + ctx.mark_dirty(volume_densities, volume_features) + + N, P, D = points_3d.shape + if D != 3: + raise ValueError("points_3d must be 3D") + if points_3d.dtype != torch.float32: + raise ValueError("points_3d must be float32") + if points_features.dtype != torch.float32: + raise ValueError("points_features must be float32") + N1, P1, C = points_features.shape + if N1 != N or P1 != P: + raise ValueError("Bad points_features shape") + if volume_densities.dtype != torch.float32: + raise ValueError("volume_densities must be float32") + N2, one, D, H, W = volume_densities.shape + if N2 != N or one != 1: + raise ValueError("Bad volume_densities shape") + if volume_features.dtype != torch.float32: + raise ValueError("volume_features must be float32") + N3, C1, D1, H1, W1 = volume_features.shape + if N3 != N or C1 != C or D1 != D or H1 != H or W1 != W: + raise ValueError("Bad volume_features shape") + if grid_sizes.dtype != torch.int64: + raise ValueError("grid_sizes must be int64") + N4, D1 = grid_sizes.shape + if N4 != N or D1 != 3: + raise ValueError("Bad grid_sizes.shape") + if mask.dtype != torch.float32: + raise ValueError("mask must be float32") + N5, P2 = mask.shape + if N5 != N or P2 != P: + raise ValueError("Bad mask shape") + + # pyre-fixme[16]: Module `pytorch3d` has no attribute `_C`. + _C.points_to_volumes_forward( + points_3d, + points_features, + volume_densities, + volume_features, + grid_sizes, + mask, + point_weight, + align_corners, + splat, + ) + if splat: + ctx.save_for_backward(points_3d, points_features, grid_sizes, mask) + else: + ctx.save_for_backward(points_3d, grid_sizes, mask) + ctx.point_weight = point_weight + ctx.splat = splat + ctx.align_corners = align_corners + return volume_densities, volume_features + + @staticmethod + @once_differentiable + def backward(ctx, grad_volume_densities, grad_volume_features): + splat = ctx.splat + N, C = grad_volume_features.shape[:2] + if splat: + points_3d, points_features, grid_sizes, mask = ctx.saved_tensors + P = points_3d.shape[1] + grad_points_3d = torch.zeros_like(points_3d) + else: + points_3d, grid_sizes, mask = ctx.saved_tensors + P = points_3d.shape[1] + ones = points_3d.new_zeros(1, 1, 1) + # There is no gradient. Just need something to let its accessors exist. + grad_points_3d = ones.expand_as(points_3d) + # points_features not needed. Just need something to let its accessors exist. + points_features = ones.expand(N, P, C) + grad_points_features = points_3d.new_zeros(N, P, C) + _C.points_to_volumes_backward( + points_3d, + points_features, + grid_sizes, + mask, + ctx.point_weight, + ctx.align_corners, + splat, + grad_volume_densities, + grad_volume_features, + grad_points_3d, + grad_points_features, + ) + + return ( + (grad_points_3d if splat else None), + grad_points_features, + grad_volume_densities, + grad_volume_features, + None, + None, + None, + None, + None, + ) + + +_points_to_volumes = _points_to_volumes_function.apply + + +def add_pointclouds_to_volumes( + pointclouds: "Pointclouds", + initial_volumes: "Volumes", + mode: str = "trilinear", + min_weight: float = 1e-4, + rescale_features: bool = True, + _python: bool = False, +) -> "Volumes": + """ + Add a batch of point clouds represented with a `Pointclouds` structure + `pointclouds` to a batch of existing volumes represented with a + `Volumes` structure `initial_volumes`. + + More specifically, the method casts a set of weighted votes (the weights are + determined based on `mode="trilinear"|"nearest"`) into the pre-initialized + `features` and `densities` fields of `initial_volumes`. + + The method returns an updated `Volumes` object that contains a copy + of `initial_volumes` with its `features` and `densities` updated with the + result of the pointcloud addition. + + Example:: + + # init a random point cloud + pointclouds = Pointclouds( + points=torch.randn(4, 100, 3), features=torch.rand(4, 100, 5) + ) + # init an empty volume centered around [0.5, 0.5, 0.5] in world coordinates + # with a voxel size of 1.0. + initial_volumes = Volumes( + features = torch.zeros(4, 5, 25, 25, 25), + densities = torch.zeros(4, 1, 25, 25, 25), + volume_translation = [-0.5, -0.5, -0.5], + voxel_size = 1.0, + ) + # add the pointcloud to the 'initial_volumes' buffer using + # trilinear splatting + updated_volumes = add_pointclouds_to_volumes( + pointclouds=pointclouds, + initial_volumes=initial_volumes, + mode="trilinear", + ) + + Args: + pointclouds: Batch of 3D pointclouds represented with a `Pointclouds` + structure. Note that `pointclouds.features` have to be defined. + initial_volumes: Batch of initial `Volumes` with pre-initialized 1-dimensional + densities which contain non-negative numbers corresponding to the + opaqueness of each voxel (the higher, the less transparent). + mode: The mode of the conversion of individual points into the volume. + Set either to `nearest` or `trilinear`: + `nearest`: Each 3D point is first rounded to the volumetric + lattice. Each voxel is then labeled with the average + over features that fall into the given voxel. + The gradients of nearest neighbor conversion w.r.t. the + 3D locations of the points in `pointclouds` are *not* defined. + `trilinear`: Each 3D point casts 8 weighted votes to the 8-neighborhood + of its floating point coordinate. The weights are + determined using a trilinear interpolation scheme. + Trilinear splatting is fully differentiable w.r.t. all input arguments. + min_weight: A scalar controlling the lowest possible total per-voxel + weight used to normalize the features accumulated in a voxel. + Only active for `mode==trilinear`. + rescale_features: If False, output features are just the sum of input and + added points. If True, they are averaged. In both cases, + output densities are just summed without rescaling, so + you may need to rescale them afterwards. + _python: Set to True to use a pure Python implementation, e.g. for test + purposes, which requires more memory and may be slower. + + Returns: + updated_volumes: Output `Volumes` structure containing the conversion result. + """ + + if len(initial_volumes) != len(pointclouds): + raise ValueError( + "'initial_volumes' and 'pointclouds' have to have the same batch size." + ) + + # obtain the features and densities + pcl_feats = pointclouds.features_padded() + pcl_3d = pointclouds.points_padded() + + if pcl_feats is None: + raise ValueError("'pointclouds' have to have their 'features' defined.") + + # obtain the conversion mask + n_per_pcl = pointclouds.num_points_per_cloud().type_as(pcl_feats) + # pyre-fixme[6]: For 1st param expected `Union[bool, float, int]` but got `Tensor`. + mask = torch.arange(n_per_pcl.max(), dtype=pcl_feats.dtype, device=pcl_feats.device) + mask = (mask[None, :] < n_per_pcl[:, None]).type_as(mask) + + # convert to the coord frame of the volume + pcl_3d_local = initial_volumes.world_to_local_coords(pcl_3d) + + features_new, densities_new = add_points_features_to_volume_densities_features( + points_3d=pcl_3d_local, + points_features=pcl_feats, + volume_features=initial_volumes.features(), + volume_densities=initial_volumes.densities(), + min_weight=min_weight, + grid_sizes=initial_volumes.get_grid_sizes(), + mask=mask, + mode=mode, + rescale_features=rescale_features, + align_corners=initial_volumes.get_align_corners(), + _python=_python, + ) + + return initial_volumes.update_padded( + new_densities=densities_new, new_features=features_new + ) + + +def add_points_features_to_volume_densities_features( + points_3d: torch.Tensor, + points_features: torch.Tensor, + volume_densities: torch.Tensor, + volume_features: Optional[torch.Tensor], + mode: str = "trilinear", + min_weight: float = 1e-4, + mask: Optional[torch.Tensor] = None, + grid_sizes: Optional[torch.LongTensor] = None, + rescale_features: bool = True, + _python: bool = False, + align_corners: bool = True, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Convert a batch of point clouds represented with tensors of per-point + 3d coordinates and their features to a batch of volumes represented + with tensors of densities and features. + + Args: + points_3d: Batch of 3D point cloud coordinates of shape + `(minibatch, N, 3)` where N is the number of points + in each point cloud. Coordinates have to be specified in the + local volume coordinates (ranging in [-1, 1]). + points_features: Features of shape `(minibatch, N, feature_dim)` corresponding + to the points of the input point clouds `pointcloud`. + volume_densities: Batch of input feature volume densities of shape + `(minibatch, 1, D, H, W)`. Each voxel should + contain a non-negative number corresponding to its + opaqueness (the higher, the less transparent). + volume_features: Batch of input feature volumes of shape + `(minibatch, feature_dim, D, H, W)` + If set to `None`, the `volume_features` will be automatically + instantiated with a correct size and filled with 0s. + mode: The mode of the conversion of individual points into the volume. + Set either to `nearest` or `trilinear`: + `nearest`: Each 3D point is first rounded to the volumetric + lattice. Each voxel is then labeled with the average + over features that fall into the given voxel. + The gradients of nearest neighbor rounding w.r.t. the + input point locations `points_3d` are *not* defined. + `trilinear`: Each 3D point casts 8 weighted votes to the 8-neighborhood + of its floating point coordinate. The weights are + determined using a trilinear interpolation scheme. + Trilinear splatting is fully differentiable w.r.t. all input arguments. + min_weight: A scalar controlling the lowest possible total per-voxel + weight used to normalize the features accumulated in a voxel. + Only active for `mode==trilinear`. + mask: A binary mask of shape `(minibatch, N)` determining which 3D points + are going to be converted to the resulting volume. + Set to `None` if all points are valid. + grid_sizes: `LongTensor` of shape (minibatch, 3) representing the + spatial resolutions of each of the the non-flattened `volumes` tensors, + or None to indicate the whole volume is used for every batch element. + rescale_features: If False, output features are just the sum of input and + added points. If True, they are averaged. In both cases, + output densities are just summed without rescaling, so + you may need to rescale them afterwards. + _python: Set to True to use a pure Python implementation. + align_corners: as for grid_sample. + Returns: + volume_features: Output volume of shape `(minibatch, feature_dim, D, H, W)` + volume_densities: Occupancy volume of shape `(minibatch, 1, D, H, W)` + containing the total amount of votes cast to each of the voxels. + """ + + # number of points in the point cloud, its dim and batch size + ba, n_points, feature_dim = points_features.shape + ba_volume, density_dim = volume_densities.shape[:2] + + if density_dim != 1: + raise ValueError("Only one-dimensional densities are allowed.") + + # init the volumetric grid sizes if uninitialized + if grid_sizes is None: + # grid sizes shape (minibatch, 3) + grid_sizes = ( + torch.LongTensor(list(volume_densities.shape[2:])) + .to(volume_densities.device) + .expand(volume_densities.shape[0], 3) + ) + + if _python: + return _add_points_features_to_volume_densities_features_python( + points_3d=points_3d, + points_features=points_features, + volume_densities=volume_densities, + volume_features=volume_features, + mode=mode, + min_weight=min_weight, + mask=mask, + # pyre-fixme[6]: For 8th param expected `LongTensor` but got `Tensor`. + grid_sizes=grid_sizes, + ) + + if mode == "trilinear": + splat = True + elif mode == "nearest": + splat = False + else: + raise ValueError('No such interpolation mode "%s"' % mode) + + if mask is None: + mask = points_3d.new_ones(1).expand(points_3d.shape[:2]) + + volume_densities, volume_features = _points_to_volumes( + points_3d, + points_features, + volume_densities, + volume_features, + grid_sizes, + 1.0, # point_weight + mask, + align_corners, # align_corners + splat, + ) + + if rescale_features: + # divide each feature by the total weight of the votes + if splat: + volume_features = volume_features / volume_densities.clamp(min_weight) + else: + volume_features = volume_features / volume_densities.clamp(1.0) + + return volume_features, volume_densities + + +def _add_points_features_to_volume_densities_features_python( + *, + points_3d: torch.Tensor, + points_features: torch.Tensor, + volume_densities: torch.Tensor, + volume_features: Optional[torch.Tensor], + mode: str, + min_weight: float, + mask: Optional[torch.Tensor], + grid_sizes: torch.LongTensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Python implementation for add_points_features_to_volume_densities_features. + + Returns: + volume_features: Output volume of shape `(minibatch, feature_dim, D, H, W)` + volume_densities: Occupancy volume of shape `(minibatch, 1, D, H, W)` + containing the total amount of votes cast to each of the voxels. + """ + ba, n_points, feature_dim = points_features.shape + + # flatten densities and features + v_shape = volume_densities.shape[2:] + volume_densities_flatten = volume_densities.view(ba, -1, 1) + n_voxels = volume_densities_flatten.shape[1] + + if volume_features is None: + # initialize features if not passed in + volume_features_flatten = volume_densities.new_zeros(ba, feature_dim, n_voxels) + else: + # otherwise just flatten + volume_features_flatten = volume_features.view(ba, feature_dim, n_voxels) + + if mode == "trilinear": # do the splatting (trilinear interp) + volume_features, volume_densities = _splat_points_to_volumes( + points_3d, + points_features, + volume_densities_flatten, + volume_features_flatten, + grid_sizes, + mask=mask, + min_weight=min_weight, + ) + elif mode == "nearest": # nearest neighbor interp + volume_features, volume_densities = _round_points_to_volumes( + points_3d, + points_features, + volume_densities_flatten, + volume_features_flatten, + grid_sizes, + mask=mask, + ) + else: + raise ValueError('No such interpolation mode "%s"' % mode) + + # reshape into the volume shape + volume_features = volume_features.view(ba, feature_dim, *v_shape) + volume_densities = volume_densities.view(ba, 1, *v_shape) + return volume_features, volume_densities + + +def _check_points_to_volumes_inputs( + points_3d: torch.Tensor, + points_features: torch.Tensor, + volume_densities: torch.Tensor, + volume_features: torch.Tensor, + grid_sizes: torch.LongTensor, + mask: Optional[torch.Tensor] = None, +) -> None: + + max_grid_size = grid_sizes.max(dim=0).values + if torch.prod(max_grid_size) > volume_densities.shape[1]: + raise ValueError( + "One of the grid sizes corresponds to a larger number" + + " of elements than the number of elements in volume_densities." + ) + + _, n_voxels, density_dim = volume_densities.shape + + if density_dim != 1: + raise ValueError("Only one-dimensional densities are allowed.") + + ba, n_points, feature_dim = points_features.shape + + if volume_features.shape[1] != feature_dim: + raise ValueError( + "volume_features have a different number of channels" + + " than points_features." + ) + + if volume_features.shape[2] != n_voxels: + raise ValueError( + "volume_features have a different number of elements" + + " than volume_densities." + ) + + +def _splat_points_to_volumes( + points_3d: torch.Tensor, + points_features: torch.Tensor, + volume_densities: torch.Tensor, + volume_features: torch.Tensor, + grid_sizes: torch.LongTensor, + min_weight: float = 1e-4, + mask: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Convert a batch of point clouds to a batch of volumes using trilinear + splatting into a volume. + + Args: + points_3d: Batch of 3D point cloud coordinates of shape + `(minibatch, N, 3)` where N is the number of points + in each point cloud. Coordinates have to be specified in the + local volume coordinates (ranging in [-1, 1]). + points_features: Features of shape `(minibatch, N, feature_dim)` + corresponding to the points of the input point cloud `points_3d`. + volume_features: Batch of input *flattened* feature volumes + of shape `(minibatch, feature_dim, N_voxels)` + volume_densities: Batch of input *flattened* feature volume densities + of shape `(minibatch, N_voxels, 1)`. Each voxel should + contain a non-negative number corresponding to its + opaqueness (the higher, the less transparent). + grid_sizes: `LongTensor` of shape (minibatch, 3) representing the + spatial resolutions of each of the the non-flattened `volumes` tensors. + Note that the following has to hold: + `torch.prod(grid_sizes, dim=1)==N_voxels` + min_weight: A scalar controlling the lowest possible total per-voxel + weight used to normalize the features accumulated in a voxel. + mask: A binary mask of shape `(minibatch, N)` determining which 3D points + are going to be converted to the resulting volume. + Set to `None` if all points are valid. + Returns: + volume_features: Output volume of shape `(minibatch, D, N_voxels)`. + volume_densities: Occupancy volume of shape `(minibatch, 1, N_voxels)` + containing the total amount of votes cast to each of the voxels. + """ + + _check_points_to_volumes_inputs( + points_3d, + points_features, + volume_densities, + volume_features, + grid_sizes, + mask=mask, + ) + + _, n_voxels, density_dim = volume_densities.shape + ba, n_points, feature_dim = points_features.shape + + # minibatch x n_points x feature_dim -> minibatch x feature_dim x n_points + points_features = points_features.permute(0, 2, 1).contiguous() + + # XYZ = the upper-left volume index of the 8-neighborhood of every point + # grid_sizes is of the form (minibatch, depth-height-width) + grid_sizes_xyz = grid_sizes[:, [2, 1, 0]] + + # Convert from points_3d in the range [-1, 1] to + # indices in the volume grid in the range [0, grid_sizes_xyz-1] + points_3d_indices = ((points_3d + 1) * 0.5) * ( + grid_sizes_xyz[:, None].type_as(points_3d) - 1 + ) + XYZ = points_3d_indices.floor().long() + rXYZ = points_3d_indices - XYZ.type_as(points_3d) # remainder of floor + + # split into separate coordinate vectors + X, Y, Z = XYZ.split(1, dim=2) + # rX = remainder after floor = 1-"the weight of each vote into + # the X coordinate of the 8-neighborhood" + rX, rY, rZ = rXYZ.split(1, dim=2) + + # get random indices for the purpose of adding out-of-bounds values + rand_idx = X.new_zeros(X.shape).random_(0, n_voxels) + + # iterate over the x, y, z indices of the 8-neighborhood (xdiff, ydiff, zdiff) + for xdiff in (0, 1): + X_ = X + xdiff + wX = (1 - xdiff) + (2 * xdiff - 1) * rX + for ydiff in (0, 1): + Y_ = Y + ydiff + wY = (1 - ydiff) + (2 * ydiff - 1) * rY + for zdiff in (0, 1): + Z_ = Z + zdiff + wZ = (1 - zdiff) + (2 * zdiff - 1) * rZ + + # weight of each vote into the given cell of 8-neighborhood + w = wX * wY * wZ + + # valid - binary indicators of votes that fall into the volume + valid = ( + (0 <= X_) + * (X_ < grid_sizes_xyz[:, None, 0:1]) + * (0 <= Y_) + * (Y_ < grid_sizes_xyz[:, None, 1:2]) + * (0 <= Z_) + * (Z_ < grid_sizes_xyz[:, None, 2:3]) + ).long() + + # linearized indices into the volume + idx = (Z_ * grid_sizes[:, None, 1:2] + Y_) * grid_sizes[ + :, None, 2:3 + ] + X_ + + # out-of-bounds features added to a random voxel idx with weight=0. + idx_valid = idx * valid + rand_idx * (1 - valid) + w_valid = w * valid.type_as(w) + if mask is not None: + w_valid = w_valid * mask.type_as(w)[:, :, None] + + # scatter add casts the votes into the weight accumulator + # and the feature accumulator + volume_densities.scatter_add_(1, idx_valid, w_valid) + + # reshape idx_valid -> (minibatch, feature_dim, n_points) + idx_valid = idx_valid.view(ba, 1, n_points).expand_as(points_features) + w_valid = w_valid.view(ba, 1, n_points) + + # volume_features of shape (minibatch, feature_dim, n_voxels) + volume_features.scatter_add_(2, idx_valid, w_valid * points_features) + + # divide each feature by the total weight of the votes + volume_features = volume_features / volume_densities.view(ba, 1, n_voxels).clamp( + min_weight + ) + + return volume_features, volume_densities + + +def _round_points_to_volumes( + points_3d: torch.Tensor, + points_features: torch.Tensor, + volume_densities: torch.Tensor, + volume_features: torch.Tensor, + grid_sizes: torch.LongTensor, + mask: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Convert a batch of point clouds to a batch of volumes using rounding to the + nearest integer coordinate of the volume. Features that fall into the same + voxel are averaged. + + Args: + points_3d: Batch of 3D point cloud coordinates of shape + `(minibatch, N, 3)` where N is the number of points + in each point cloud. Coordinates have to be specified in the + local volume coordinates (ranging in [-1, 1]). + points_features: Features of shape `(minibatch, N, feature_dim)` + corresponding to the points of the input point cloud `points_3d`. + volume_features: Batch of input *flattened* feature volumes + of shape `(minibatch, feature_dim, N_voxels)` + volume_densities: Batch of input *flattened* feature volume densities + of shape `(minibatch, 1, N_voxels)`. Each voxel should + contain a non-negative number corresponding to its + opaqueness (the higher, the less transparent). + grid_sizes: `LongTensor` of shape (minibatch, 3) representing the + spatial resolutions of each of the the non-flattened `volumes` tensors. + Note that the following has to hold: + `torch.prod(grid_sizes, dim=1)==N_voxels` + mask: A binary mask of shape `(minibatch, N)` determining which 3D points + are going to be converted to the resulting volume. + Set to `None` if all points are valid. + Returns: + volume_features: Output volume of shape `(minibatch, D, N_voxels)`. + volume_densities: Occupancy volume of shape `(minibatch, 1, N_voxels)` + containing the total amount of votes cast to each of the voxels. + """ + + _check_points_to_volumes_inputs( + points_3d, + points_features, + volume_densities, + volume_features, + grid_sizes, + mask=mask, + ) + + _, n_voxels, density_dim = volume_densities.shape + ba, n_points, feature_dim = points_features.shape + + # minibatch x n_points x feature_dim-> minibatch x feature_dim x n_points + points_features = points_features.permute(0, 2, 1).contiguous() + + # round the coordinates to nearest integer + # grid_sizes is of the form (minibatch, depth-height-width) + grid_sizes_xyz = grid_sizes[:, [2, 1, 0]] + XYZ = ((points_3d.detach() + 1) * 0.5) * ( + grid_sizes_xyz[:, None].type_as(points_3d) - 1 + ) + XYZ = torch.round(XYZ).long() + + # split into separate coordinate vectors + X, Y, Z = XYZ.split(1, dim=2) + + # valid - binary indicators of votes that fall into the volume + # pyre-fixme[9]: grid_sizes has type `LongTensor`; used as `Tensor`. + grid_sizes = grid_sizes.type_as(XYZ) + valid = ( + (0 <= X) + * (X < grid_sizes_xyz[:, None, 0:1]) + * (0 <= Y) + * (Y < grid_sizes_xyz[:, None, 1:2]) + * (0 <= Z) + * (Z < grid_sizes_xyz[:, None, 2:3]) + ).long() + if mask is not None: + valid = valid * mask[:, :, None].long() + + # get random indices for the purpose of adding out-of-bounds values + rand_idx = valid.new_zeros(X.shape).random_(0, n_voxels) + + # linearized indices into the volume + idx = (Z * grid_sizes[:, None, 1:2] + Y) * grid_sizes[:, None, 2:3] + X + + # out-of-bounds features added to a random voxel idx with weight=0. + idx_valid = idx * valid + rand_idx * (1 - valid) + w_valid = valid.type_as(volume_features) + + # scatter add casts the votes into the weight accumulator + # and the feature accumulator + volume_densities.scatter_add_(1, idx_valid, w_valid) + + # reshape idx_valid -> (minibatch, feature_dim, n_points) + idx_valid = idx_valid.view(ba, 1, n_points).expand_as(points_features) + w_valid = w_valid.view(ba, 1, n_points) + + # volume_features of shape (minibatch, feature_dim, n_voxels) + volume_features.scatter_add_(2, idx_valid, w_valid * points_features) + + # divide each feature by the total weight of the votes + volume_features = volume_features / volume_densities.view(ba, 1, n_voxels).clamp( + 1.0 + ) + + return volume_features, volume_densities diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/sample_farthest_points.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/sample_farthest_points.py new file mode 100644 index 0000000000000000000000000000000000000000..a45b1de229d7640fa6fbadb98e036c8f390939c0 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/sample_farthest_points.py @@ -0,0 +1,197 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from random import randint +from typing import List, Optional, Tuple, Union + +import torch +from pytorch3d import _C + +from .utils import masked_gather + + +def sample_farthest_points( + points: torch.Tensor, + lengths: Optional[torch.Tensor] = None, + K: Union[int, List, torch.Tensor] = 50, + random_start_point: bool = False, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Iterative farthest point sampling algorithm [1] to subsample a set of + K points from a given pointcloud. At each iteration, a point is selected + which has the largest nearest neighbor distance to any of the + already selected points. + + Farthest point sampling provides more uniform coverage of the input + point cloud compared to uniform random sampling. + + [1] Charles R. Qi et al, "PointNet++: Deep Hierarchical Feature Learning + on Point Sets in a Metric Space", NeurIPS 2017. + + Args: + points: (N, P, D) array containing the batch of pointclouds + lengths: (N,) number of points in each pointcloud (to support heterogeneous + batches of pointclouds) + K: samples required in each sampled point cloud (this is typically << P). If + K is an int then the same number of samples are selected for each + pointcloud in the batch. If K is a tensor is should be length (N,) + giving the number of samples to select for each element in the batch + random_start_point: bool, if True, a random point is selected as the starting + point for iterative sampling. + + Returns: + selected_points: (N, K, D), array of selected values from points. If the input + K is a tensor, then the shape will be (N, max(K), D), and padded with + 0.0 for batch elements where k_i < max(K). + selected_indices: (N, K) array of selected indices. If the input + K is a tensor, then the shape will be (N, max(K), D), and padded with + -1 for batch elements where k_i < max(K). + """ + N, P, D = points.shape + device = points.device + + # Validate inputs + if lengths is None: + lengths = torch.full((N,), P, dtype=torch.int64, device=device) + else: + if lengths.shape != (N,): + raise ValueError("points and lengths must have same batch dimension.") + if lengths.max() > P: + raise ValueError("A value in lengths was too large.") + + # TODO: support providing K as a ratio of the total number of points instead of as an int + if isinstance(K, int): + K = torch.full((N,), K, dtype=torch.int64, device=device) + elif isinstance(K, list): + K = torch.tensor(K, dtype=torch.int64, device=device) + + if K.shape[0] != N: + raise ValueError("K and points must have the same batch dimension") + + # Check dtypes are correct and convert if necessary + if not (points.dtype == torch.float32): + points = points.to(torch.float32) + if not (lengths.dtype == torch.int64): + lengths = lengths.to(torch.int64) + if not (K.dtype == torch.int64): + K = K.to(torch.int64) + + # Generate the starting indices for sampling + start_idxs = torch.zeros_like(lengths) + if random_start_point: + for n in range(N): + # pyre-fixme[6]: For 1st param expected `int` but got `Tensor`. + start_idxs[n] = torch.randint(high=lengths[n], size=(1,)).item() + + with torch.no_grad(): + # pyre-fixme[16]: `pytorch3d_._C` has no attribute `sample_farthest_points`. + idx = _C.sample_farthest_points(points, lengths, K, start_idxs) + sampled_points = masked_gather(points, idx) + + return sampled_points, idx + + +def sample_farthest_points_naive( + points: torch.Tensor, + lengths: Optional[torch.Tensor] = None, + K: Union[int, List, torch.Tensor] = 50, + random_start_point: bool = False, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Same Args/Returns as sample_farthest_points + """ + N, P, D = points.shape + device = points.device + + # Validate inputs + if lengths is None: + lengths = torch.full((N,), P, dtype=torch.int64, device=device) + else: + if lengths.shape != (N,): + raise ValueError("points and lengths must have same batch dimension.") + if lengths.max() > P: + raise ValueError("Invalid lengths.") + + # TODO: support providing K as a ratio of the total number of points instead of as an int + if isinstance(K, int): + K = torch.full((N,), K, dtype=torch.int64, device=device) + elif isinstance(K, list): + K = torch.tensor(K, dtype=torch.int64, device=device) + + if K.shape[0] != N: + raise ValueError("K and points must have the same batch dimension") + + # Find max value of K + max_K = torch.max(K) + + # List of selected indices from each batch element + all_sampled_indices = [] + + for n in range(N): + # Initialize an array for the sampled indices, shape: (max_K,) + sample_idx_batch = torch.full( + # pyre-fixme[6]: For 1st param expected `Union[List[int], Size, + # typing.Tuple[int, ...]]` but got `Tuple[Tensor]`. + (max_K,), + fill_value=-1, + dtype=torch.int64, + device=device, + ) + + # Initialize closest distances to inf, shape: (P,) + # This will be updated at each iteration to track the closest distance of the + # remaining points to any of the selected points + closest_dists = points.new_full( + # pyre-fixme[6]: For 1st param expected `Union[List[int], Size, + # typing.Tuple[int, ...]]` but got `Tuple[Tensor]`. + (lengths[n],), + float("inf"), + dtype=torch.float32, + ) + + # Select a random point index and save it as the starting point + # pyre-fixme[6]: For 2nd argument expected `int` but got `Tensor`. + selected_idx = randint(0, lengths[n] - 1) if random_start_point else 0 + sample_idx_batch[0] = selected_idx + + # If the pointcloud has fewer than K points then only iterate over the min + # pyre-fixme[6]: For 1st param expected `SupportsRichComparisonT` but got + # `Tensor`. + # pyre-fixme[6]: For 2nd param expected `SupportsRichComparisonT` but got + # `Tensor`. + k_n = min(lengths[n], K[n]) + + # Iteratively select points for a maximum of k_n + for i in range(1, k_n): + # Find the distance between the last selected point + # and all the other points. If a point has already been selected + # it's distance will be 0.0 so it will not be selected again as the max. + dist = points[n, selected_idx, :] - points[n, : lengths[n], :] + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and + # `int`. + dist_to_last_selected = (dist**2).sum(-1) # (P - i) + + # If closer than currently saved distance to one of the selected + # points, then updated closest_dists + closest_dists = torch.min(dist_to_last_selected, closest_dists) # (P - i) + + # The aim is to pick the point that has the largest + # nearest neighbour distance to any of the already selected points + selected_idx = torch.argmax(closest_dists) + sample_idx_batch[i] = selected_idx + + # Add the list of points for this batch to the final list + all_sampled_indices.append(sample_idx_batch) + + all_sampled_indices = torch.stack(all_sampled_indices, dim=0) + + # Gather the points + all_sampled_points = masked_gather(points, all_sampled_indices) + + # Return (N, max_K, D) subsampled points and indices + return all_sampled_points, all_sampled_indices diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/sample_points_from_meshes.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/sample_points_from_meshes.py new file mode 100644 index 0000000000000000000000000000000000000000..125fe683db288b8effbc0845066faba9287cbf94 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/sample_points_from_meshes.py @@ -0,0 +1,179 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +""" +This module implements utility functions for sampling points from +batches of meshes. +""" +import sys +from typing import Tuple, Union + +import torch + +from pytorch3d.ops.mesh_face_areas_normals import mesh_face_areas_normals + +from pytorch3d.ops.packed_to_padded import packed_to_padded +from pytorch3d.renderer.mesh.rasterizer import Fragments as MeshFragments + + +def sample_points_from_meshes( + meshes, + num_samples: int = 10000, + return_normals: bool = False, + return_textures: bool = False, +) -> Union[ + torch.Tensor, + Tuple[torch.Tensor, torch.Tensor], + Tuple[torch.Tensor, torch.Tensor, torch.Tensor], +]: + """ + Convert a batch of meshes to a batch of pointclouds by uniformly sampling + points on the surface of the mesh with probability proportional to the + face area. + + Args: + meshes: A Meshes object with a batch of N meshes. + num_samples: Integer giving the number of point samples per mesh. + return_normals: If True, return normals for the sampled points. + return_textures: If True, return textures for the sampled points. + + Returns: + 3-element tuple containing + + - **samples**: FloatTensor of shape (N, num_samples, 3) giving the + coordinates of sampled points for each mesh in the batch. For empty + meshes the corresponding row in the samples array will be filled with 0. + - **normals**: FloatTensor of shape (N, num_samples, 3) giving a normal vector + to each sampled point. Only returned if return_normals is True. + For empty meshes the corresponding row in the normals array will + be filled with 0. + - **textures**: FloatTensor of shape (N, num_samples, C) giving a C-dimensional + texture vector to each sampled point. Only returned if return_textures is True. + For empty meshes the corresponding row in the textures array will + be filled with 0. + + Note that in a future releases, we will replace the 3-element tuple output + with a `Pointclouds` datastructure, as follows + + .. code-block:: python + + Pointclouds(samples, normals=normals, features=textures) + """ + if meshes.isempty(): + raise ValueError("Meshes are empty.") + + verts = meshes.verts_packed() + if not torch.isfinite(verts).all(): + raise ValueError("Meshes contain nan or inf.") + + if return_textures and meshes.textures is None: + raise ValueError("Meshes do not contain textures.") + + faces = meshes.faces_packed() + mesh_to_face = meshes.mesh_to_faces_packed_first_idx() + num_meshes = len(meshes) + num_valid_meshes = torch.sum(meshes.valid) # Non empty meshes. + + # Initialize samples tensor with fill value 0 for empty meshes. + samples = torch.zeros((num_meshes, num_samples, 3), device=meshes.device) + + # Only compute samples for non empty meshes + with torch.no_grad(): + areas, _ = mesh_face_areas_normals(verts, faces) # Face areas can be zero. + max_faces = meshes.num_faces_per_mesh().max().item() + areas_padded = packed_to_padded( + areas, mesh_to_face[meshes.valid], max_faces + ) # (N, F) + + # TODO (gkioxari) Confirm multinomial bug is not present with real data. + sample_face_idxs = areas_padded.multinomial( + num_samples, replacement=True + ) # (N, num_samples) + sample_face_idxs += mesh_to_face[meshes.valid].view(num_valid_meshes, 1) + + # Get the vertex coordinates of the sampled faces. + face_verts = verts[faces] + v0, v1, v2 = face_verts[:, 0], face_verts[:, 1], face_verts[:, 2] + + # Randomly generate barycentric coords. + w0, w1, w2 = _rand_barycentric_coords( + num_valid_meshes, num_samples, verts.dtype, verts.device + ) + + # Use the barycentric coords to get a point on each sampled face. + a = v0[sample_face_idxs] # (N, num_samples, 3) + b = v1[sample_face_idxs] + c = v2[sample_face_idxs] + samples[meshes.valid] = w0[:, :, None] * a + w1[:, :, None] * b + w2[:, :, None] * c + + if return_normals: + # Initialize normals tensor with fill value 0 for empty meshes. + # Normals for the sampled points are face normals computed from + # the vertices of the face in which the sampled point lies. + normals = torch.zeros((num_meshes, num_samples, 3), device=meshes.device) + vert_normals = (v1 - v0).cross(v2 - v1, dim=1) + vert_normals = vert_normals / vert_normals.norm(dim=1, p=2, keepdim=True).clamp( + min=sys.float_info.epsilon + ) + vert_normals = vert_normals[sample_face_idxs] + normals[meshes.valid] = vert_normals + + if return_textures: + # fragment data are of shape NxHxWxK. Here H=S, W=1 & K=1. + pix_to_face = sample_face_idxs.view(len(meshes), num_samples, 1, 1) # NxSx1x1 + bary = torch.stack((w0, w1, w2), dim=2).unsqueeze(2).unsqueeze(2) # NxSx1x1x3 + # zbuf and dists are not used in `sample_textures` so we initialize them with dummy + dummy = torch.zeros( + (len(meshes), num_samples, 1, 1), device=meshes.device, dtype=torch.float32 + ) # NxSx1x1 + fragments = MeshFragments( + pix_to_face=pix_to_face, zbuf=dummy, bary_coords=bary, dists=dummy + ) + textures = meshes.sample_textures(fragments) # NxSx1x1xC + textures = textures[:, :, 0, 0, :] # NxSxC + + # return + # TODO(gkioxari) consider returning a Pointclouds instance [breaking] + if return_normals and return_textures: + # pyre-fixme[61]: `normals` may not be initialized here. + # pyre-fixme[61]: `textures` may not be initialized here. + return samples, normals, textures + if return_normals: # return_textures is False + # pyre-fixme[61]: `normals` may not be initialized here. + return samples, normals + if return_textures: # return_normals is False + # pyre-fixme[61]: `textures` may not be initialized here. + return samples, textures + return samples + + +def _rand_barycentric_coords( + size1, size2, dtype: torch.dtype, device: torch.device +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Helper function to generate random barycentric coordinates which are uniformly + distributed over a triangle. + + Args: + size1, size2: The number of coordinates generated will be size1*size2. + Output tensors will each be of shape (size1, size2). + dtype: Datatype to generate. + device: A torch.device object on which the outputs will be allocated. + + Returns: + w0, w1, w2: Tensors of shape (size1, size2) giving random barycentric + coordinates + """ + uv = torch.rand(2, size1, size2, dtype=dtype, device=device) + u, v = uv[0], uv[1] + u_sqrt = u.sqrt() + w0 = 1.0 - u_sqrt + w1 = u_sqrt * (1.0 - v) + w2 = u_sqrt * v + return w0, w1, w2 diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/subdivide_meshes.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/subdivide_meshes.py new file mode 100644 index 0000000000000000000000000000000000000000..86204309a4a79896028ba82c9df0df273c3c15b4 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/subdivide_meshes.py @@ -0,0 +1,472 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +import torch +import torch.nn as nn +from pytorch3d.structures import Meshes + + +class SubdivideMeshes(nn.Module): + """ + Subdivide a triangle mesh by adding a new vertex at the center of each edge + and dividing each face into four new faces. Vectors of vertex + attributes can also be subdivided by averaging the values of the attributes + at the two vertices which form each edge. This implementation + preserves face orientation - if the vertices of a face are all ordered + counter-clockwise, then the faces in the subdivided meshes will also have + their vertices ordered counter-clockwise. + + If meshes is provided as an input, the initializer performs the relatively + expensive computation of determining the new face indices. This one-time + computation can be reused for all meshes with the same face topology + but different vertex positions. + """ + + def __init__(self, meshes=None) -> None: + """ + Args: + meshes: Meshes object or None. If a meshes object is provided, + the first mesh is used to compute the new faces of the + subdivided topology which can be reused for meshes with + the same input topology. + """ + super(SubdivideMeshes, self).__init__() + + self.precomputed = False + self._N = -1 + if meshes is not None: + # This computation is on indices, so gradients do not need to be + # tracked. + mesh = meshes[0] + with torch.no_grad(): + subdivided_faces = self.subdivide_faces(mesh) + if subdivided_faces.shape[1] != 3: + raise ValueError("faces can only have three vertices") + self.register_buffer("_subdivided_faces", subdivided_faces) + self.precomputed = True + + def subdivide_faces(self, meshes): + r""" + Args: + meshes: a Meshes object. + + Returns: + subdivided_faces_packed: (4*sum(F_n), 3) shape LongTensor of + original and new faces. + + Refer to pytorch3d.structures.meshes.py for more details on packed + representations of faces. + + Each face is split into 4 faces e.g. Input face + :: + v0 + /\ + / \ + / \ + e1 / \ e0 + / \ + / \ + / \ + /______________\ + v2 e2 v1 + + faces_packed = [[0, 1, 2]] + faces_packed_to_edges_packed = [[2, 1, 0]] + + `faces_packed_to_edges_packed` is used to represent all the new + vertex indices corresponding to the mid-points of edges in the mesh. + The actual vertex coordinates will be computed in the forward function. + To get the indices of the new vertices, offset + `faces_packed_to_edges_packed` by the total number of vertices. + :: + faces_packed_to_edges_packed = [[2, 1, 0]] + 3 = [[5, 4, 3]] + + e.g. subdivided face + :: + v0 + /\ + / \ + / f0 \ + v4 /______\ v3 + /\ /\ + / \ f3 / \ + / f2 \ / f1 \ + /______\/______\ + v2 v5 v1 + + f0 = [0, 3, 4] + f1 = [1, 5, 3] + f2 = [2, 4, 5] + f3 = [5, 4, 3] + + """ + verts_packed = meshes.verts_packed() + with torch.no_grad(): + faces_packed = meshes.faces_packed() + faces_packed_to_edges_packed = ( + meshes.faces_packed_to_edges_packed() + verts_packed.shape[0] + ) + + f0 = torch.stack( + [ + faces_packed[:, 0], + faces_packed_to_edges_packed[:, 2], + faces_packed_to_edges_packed[:, 1], + ], + dim=1, + ) + f1 = torch.stack( + [ + faces_packed[:, 1], + faces_packed_to_edges_packed[:, 0], + faces_packed_to_edges_packed[:, 2], + ], + dim=1, + ) + f2 = torch.stack( + [ + faces_packed[:, 2], + faces_packed_to_edges_packed[:, 1], + faces_packed_to_edges_packed[:, 0], + ], + dim=1, + ) + f3 = faces_packed_to_edges_packed + subdivided_faces_packed = torch.cat( + [f0, f1, f2, f3], dim=0 + ) # (4*sum(F_n), 3) + + return subdivided_faces_packed + + def forward(self, meshes, feats=None): + """ + Subdivide a batch of meshes by adding a new vertex on each edge, and + dividing each face into four new faces. New meshes contains two types + of vertices: + 1) Vertices that appear in the input meshes. + Data for these vertices are copied from the input meshes. + 2) New vertices at the midpoint of each edge. + Data for these vertices is the average of the data for the two + vertices that make up the edge. + + Args: + meshes: Meshes object representing a batch of meshes. + feats: Per-vertex features to be subdivided along with the verts. + Should be parallel to the packed vert representation of the + input meshes; so it should have shape (V, D) where V is the + total number of verts in the input meshes. Default: None. + + Returns: + 2-element tuple containing + + - **new_meshes**: Meshes object of a batch of subdivided meshes. + - **new_feats**: (optional) Tensor of subdivided feats, parallel to the + (packed) vertices of the subdivided meshes. Only returned + if feats is not None. + + """ + self._N = len(meshes) + if self.precomputed: + return self.subdivide_homogeneous(meshes, feats) + else: + return self.subdivide_heterogenerous(meshes, feats) + + def subdivide_homogeneous(self, meshes, feats=None): + """ + Subdivide verts (and optionally features) of a batch of meshes + where each mesh has the same topology of faces. The subdivided faces + are precomputed in the initializer. + + Args: + meshes: Meshes object representing a batch of meshes. + feats: Per-vertex features to be subdivided along with the verts. + + Returns: + 2-element tuple containing + + - **new_meshes**: Meshes object of a batch of subdivided meshes. + - **new_feats**: (optional) Tensor of subdivided feats, parallel to the + (packed) vertices of the subdivided meshes. Only returned + if feats is not None. + """ + verts = meshes.verts_padded() # (N, V, D) + edges = meshes[0].edges_packed() + + # The set of faces is the same across the different meshes. + new_faces = self._subdivided_faces.view(1, -1, 3).expand(self._N, -1, -1) + + # Add one new vertex at the midpoint of each edge by taking the average + # of the vertices that form each edge. + new_verts = verts[:, edges].mean(dim=2) + new_verts = torch.cat([verts, new_verts], dim=1) # (sum(V_n)+sum(E_n), 3) + new_feats = None + + # Calculate features for new vertices. + if feats is not None: + if feats.dim() == 2: + # feats is in packed format, transform it from packed to + # padded, i.e. (N*V, D) to (N, V, D). + feats = feats.view(verts.size(0), verts.size(1), feats.size(1)) + if feats.dim() != 3: + raise ValueError("features need to be of shape (N, V, D) or (N*V, D)") + + # Take average of the features at the vertices that form each edge. + new_feats = feats[:, edges].mean(dim=2) + new_feats = torch.cat([feats, new_feats], dim=1) # (sum(V_n)+sum(E_n), 3) + + new_meshes = Meshes(verts=new_verts, faces=new_faces) + + if feats is None: + return new_meshes + else: + return new_meshes, new_feats + + def subdivide_heterogenerous(self, meshes, feats=None): + """ + Subdivide faces, verts (and optionally features) of a batch of meshes + where each mesh can have different face topologies. + + Args: + meshes: Meshes object representing a batch of meshes. + feats: Per-vertex features to be subdivided along with the verts. + + Returns: + 2-element tuple containing + + - **new_meshes**: Meshes object of a batch of subdivided meshes. + - **new_feats**: (optional) Tensor of subdivided feats, parallel to the + (packed) vertices of the subdivided meshes. Only returned + if feats is not None. + """ + + # The computation of new faces is on face indices, so gradients do not + # need to be tracked. + verts = meshes.verts_packed() + with torch.no_grad(): + new_faces = self.subdivide_faces(meshes) + edges = meshes.edges_packed() + face_to_mesh_idx = meshes.faces_packed_to_mesh_idx() + edge_to_mesh_idx = meshes.edges_packed_to_mesh_idx() + num_edges_per_mesh = edge_to_mesh_idx.bincount(minlength=self._N) + num_verts_per_mesh = meshes.num_verts_per_mesh() + num_faces_per_mesh = meshes.num_faces_per_mesh() + + # Add one new vertex at the midpoint of each edge. + new_verts_per_mesh = num_verts_per_mesh + num_edges_per_mesh # (N,) + new_face_to_mesh_idx = torch.cat([face_to_mesh_idx] * 4, dim=0) + + # Calculate the indices needed to group the new and existing verts + # for each mesh. + verts_sort_idx = _create_verts_index( + num_verts_per_mesh, num_edges_per_mesh, meshes.device + ) # (sum(V_n)+sum(E_n),) + + verts_ordered_idx_init = torch.zeros( + new_verts_per_mesh.sum(), dtype=torch.int64, device=meshes.device + ) # (sum(V_n)+sum(E_n),) + + # Reassign vertex indices so that existing and new vertices for each + # mesh are sequential. + verts_ordered_idx = verts_ordered_idx_init.scatter_add( + 0, + verts_sort_idx, + torch.arange(new_verts_per_mesh.sum(), device=meshes.device), + ) + + # Retrieve vertex indices for each face. + new_faces = verts_ordered_idx[new_faces] + + # Calculate the indices needed to group the existing and new faces + # for each mesh. + face_sort_idx = _create_faces_index( + num_faces_per_mesh, device=meshes.device + ) + + # Reorder the faces to sequentially group existing and new faces + # for each mesh. + new_faces = new_faces[face_sort_idx] + new_face_to_mesh_idx = new_face_to_mesh_idx[face_sort_idx] + new_faces_per_mesh = new_face_to_mesh_idx.bincount( + minlength=self._N + ) # (sum(F_n)*4) + + # Add one new vertex at the midpoint of each edge by taking the average + # of the verts that form each edge. + new_verts = verts[edges].mean(dim=1) + new_verts = torch.cat([verts, new_verts], dim=0) + + # Reorder the verts to sequentially group existing and new verts for + # each mesh. + new_verts = new_verts[verts_sort_idx] + + if feats is not None: + new_feats = feats[edges].mean(dim=1) + new_feats = torch.cat([feats, new_feats], dim=0) + new_feats = new_feats[verts_sort_idx] + + verts_list = list(new_verts.split(new_verts_per_mesh.tolist(), 0)) + faces_list = list(new_faces.split(new_faces_per_mesh.tolist(), 0)) + new_verts_per_mesh_cumsum = torch.cat( + [ + new_verts_per_mesh.new_full(size=(1,), fill_value=0.0), + new_verts_per_mesh.cumsum(0)[:-1], + ], + dim=0, + ) + faces_list = [ + faces_list[n] - new_verts_per_mesh_cumsum[n] for n in range(self._N) + ] + if feats is not None: + feats_list = new_feats.split(new_verts_per_mesh.tolist(), 0) + new_meshes = Meshes(verts=verts_list, faces=faces_list) + + if feats is None: + return new_meshes + else: + new_feats = torch.cat(feats_list, dim=0) + return new_meshes, new_feats + + +def _create_verts_index(verts_per_mesh, edges_per_mesh, device=None): + """ + Helper function to group the vertex indices for each mesh. New vertices are + stacked at the end of the original verts tensor, so in order to have + sequential packing, the verts tensor needs to be reordered so that the + vertices corresponding to each mesh are grouped together. + + Args: + verts_per_mesh: Tensor of shape (N,) giving the number of vertices + in each mesh in the batch where N is the batch size. + edges_per_mesh: Tensor of shape (N,) giving the number of edges + in each mesh in the batch + + Returns: + verts_idx: A tensor with vert indices for each mesh ordered sequentially + by mesh index. + """ + # e.g. verts_per_mesh = (4, 5, 6) + # e.g. edges_per_mesh = (5, 7, 9) + + V = verts_per_mesh.sum() # e.g. 15 + E = edges_per_mesh.sum() # e.g. 21 + + verts_per_mesh_cumsum = verts_per_mesh.cumsum(dim=0) # (N,) e.g. (4, 9, 15) + edges_per_mesh_cumsum = edges_per_mesh.cumsum(dim=0) # (N,) e.g. (5, 12, 21) + + v_to_e_idx = verts_per_mesh_cumsum.clone() + + # vertex to edge index. + v_to_e_idx[1:] += edges_per_mesh_cumsum[ + :-1 + ] # e.g. (4, 9, 15) + (0, 5, 12) = (4, 14, 27) + + # vertex to edge offset. + v_to_e_offset = V - verts_per_mesh_cumsum # e.g. 15 - (4, 9, 15) = (11, 6, 0) + v_to_e_offset[1:] += edges_per_mesh_cumsum[ + :-1 + ] # e.g. (11, 6, 0) + (0, 5, 12) = (11, 11, 12) + e_to_v_idx = ( + verts_per_mesh_cumsum[:-1] + edges_per_mesh_cumsum[:-1] + ) # (4, 9) + (5, 12) = (9, 21) + e_to_v_offset = ( + verts_per_mesh_cumsum[:-1] - edges_per_mesh_cumsum[:-1] - V + ) # (4, 9) - (5, 12) - 15 = (-16, -18) + + # Add one new vertex per edge. + idx_diffs = torch.ones(V + E, device=device, dtype=torch.int64) # (36,) + idx_diffs[v_to_e_idx] += v_to_e_offset + idx_diffs[e_to_v_idx] += e_to_v_offset + + # e.g. + # [ + # 1, 1, 1, 1, 12, 1, 1, 1, 1, + # -15, 1, 1, 1, 1, 12, 1, 1, 1, 1, 1, 1, + # -17, 1, 1, 1, 1, 1, 13, 1, 1, 1, 1, 1, 1, 1 + # ] + + verts_idx = idx_diffs.cumsum(dim=0) - 1 + + # e.g. + # [ + # 0, 1, 2, 3, 15, 16, 17, 18, 19, --> mesh 0 + # 4, 5, 6, 7, 8, 20, 21, 22, 23, 24, 25, 26, --> mesh 1 + # 9, 10, 11, 12, 13, 14, 27, 28, 29, 30, 31, 32, 33, 34, 35 --> mesh 2 + # ] + # where for mesh 0, [0, 1, 2, 3] are the indices of the existing verts, and + # [15, 16, 17, 18, 19] are the indices of the new verts after subdivision. + + return verts_idx + + +def _create_faces_index(faces_per_mesh: torch.Tensor, device=None): + """ + Helper function to group the faces indices for each mesh. New faces are + stacked at the end of the original faces tensor, so in order to have + sequential packing, the faces tensor needs to be reordered to that faces + corresponding to each mesh are grouped together. + + Args: + faces_per_mesh: Tensor of shape (N,) giving the number of faces + in each mesh in the batch where N is the batch size. + + Returns: + faces_idx: A tensor with face indices for each mesh ordered sequentially + by mesh index. + """ + # e.g. faces_per_mesh = [2, 5, 3] + + F = faces_per_mesh.sum() # e.g. 10 + faces_per_mesh_cumsum = faces_per_mesh.cumsum(dim=0) # (N,) e.g. (2, 7, 10) + + switch1_idx = faces_per_mesh_cumsum.clone() + switch1_idx[1:] += ( + 3 * faces_per_mesh_cumsum[:-1] + ) # e.g. (2, 7, 10) + (0, 6, 21) = (2, 13, 31) + + switch2_idx = 2 * faces_per_mesh_cumsum # e.g. (4, 14, 20) + switch2_idx[1:] += ( + 2 * faces_per_mesh_cumsum[:-1] + ) # e.g. (4, 14, 20) + (0, 4, 14) = (4, 18, 34) + + switch3_idx = 3 * faces_per_mesh_cumsum # e.g. (6, 21, 30) + switch3_idx[1:] += faces_per_mesh_cumsum[ + :-1 + ] # e.g. (6, 21, 30) + (0, 2, 7) = (6, 23, 37) + + switch4_idx = 4 * faces_per_mesh_cumsum[:-1] # e.g. (8, 28) + + switch123_offset = F - faces_per_mesh # e.g. (8, 5, 7) + + # pyre-fixme[6]: For 1st param expected `Union[List[int], Size, + # typing.Tuple[int, ...]]` but got `Tensor`. + idx_diffs = torch.ones(4 * F, device=device, dtype=torch.int64) + idx_diffs[switch1_idx] += switch123_offset + idx_diffs[switch2_idx] += switch123_offset + idx_diffs[switch3_idx] += switch123_offset + idx_diffs[switch4_idx] -= 3 * F + + # e.g + # [ + # 1, 1, 9, 1, 9, 1, 9, 1, -> mesh 0 + # -29, 1, 1, 1, 1, 6, 1, 1, 1, 1, 6, 1, 1, 1, 1, 6, 1, 1, 1, 1, -> mesh 1 + # -29, 1, 1, 8, 1, 1, 8, 1, 1, 8, 1, 1 -> mesh 2 + # ] + + faces_idx = idx_diffs.cumsum(dim=0) - 1 + + # e.g. + # [ + # 0, 1, 10, 11, 20, 21, 30, 31, + # 2, 3, 4, 5, 6, 12, 13, 14, 15, 16, 22, 23, 24, 25, 26, 32, 33, 34, 35, 36, + # 7, 8, 9, 17, 18, 19, 27, 28, 29, 37, 38, 39 + # ] + # where for mesh 0, [0, 1] are the indices of the existing faces, and + # [10, 11, 20, 21, 30, 31] are the indices of the new faces after subdivision. + + return faces_idx diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/utils.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..29afc3f9673bd2d81501464fa8a47edd01581f30 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/utils.py @@ -0,0 +1,207 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from typing import Optional, Tuple, TYPE_CHECKING, Union + +import torch + +from .knn import knn_points + + +if TYPE_CHECKING: + from pytorch3d.structures import Pointclouds + + +def masked_gather(points: torch.Tensor, idx: torch.Tensor) -> torch.Tensor: + """ + Helper function for torch.gather to collect the points at + the given indices in idx where some of the indices might be -1 to + indicate padding. These indices are first replaced with 0. + Then the points are gathered after which the padded values + are set to 0.0. + + Args: + points: (N, P, D) float32 tensor of points + idx: (N, K) or (N, P, K) long tensor of indices into points, where + some indices are -1 to indicate padding + + Returns: + selected_points: (N, K, D) float32 tensor of points + at the given indices + """ + + if len(idx) != len(points): + raise ValueError("points and idx must have the same batch dimension") + + N, P, D = points.shape + + if idx.ndim == 3: + # Case: KNN, Ball Query where idx is of shape (N, P', K) + # where P' is not necessarily the same as P as the + # points may be gathered from a different pointcloud. + K = idx.shape[2] + # Match dimensions for points and indices + idx_expanded = idx[..., None].expand(-1, -1, -1, D) + points = points[:, :, None, :].expand(-1, -1, K, -1) + elif idx.ndim == 2: + # Farthest point sampling where idx is of shape (N, K) + idx_expanded = idx[..., None].expand(-1, -1, D) + else: + raise ValueError("idx format is not supported %s" % repr(idx.shape)) + + idx_expanded_mask = idx_expanded.eq(-1) + idx_expanded = idx_expanded.clone() + # Replace -1 values with 0 for gather + idx_expanded[idx_expanded_mask] = 0 + # Gather points + selected_points = points.gather(dim=1, index=idx_expanded) + # Replace padded values + selected_points[idx_expanded_mask] = 0.0 + return selected_points + + +def wmean( + x: torch.Tensor, + weight: Optional[torch.Tensor] = None, + dim: Union[int, Tuple[int]] = -2, + keepdim: bool = True, + eps: float = 1e-9, +) -> torch.Tensor: + """ + Finds the mean of the input tensor across the specified dimension. + If the `weight` argument is provided, computes weighted mean. + Args: + x: tensor of shape `(*, D)`, where D is assumed to be spatial; + weights: if given, non-negative tensor of shape `(*,)`. It must be + broadcastable to `x.shape[:-1]`. Note that the weights for + the last (spatial) dimension are assumed same; + dim: dimension(s) in `x` to average over; + keepdim: tells whether to keep the resulting singleton dimension. + eps: minimum clamping value in the denominator. + Returns: + the mean tensor: + * if `weights` is None => `mean(x, dim)`, + * otherwise => `sum(x*w, dim) / max{sum(w, dim), eps}`. + """ + args = {"dim": dim, "keepdim": keepdim} + + if weight is None: + # pyre-fixme[6]: For 1st param expected `Optional[dtype]` but got + # `Union[Tuple[int], int]`. + return x.mean(**args) + + if any( + xd != wd and xd != 1 and wd != 1 + for xd, wd in zip(x.shape[-2::-1], weight.shape[::-1]) + ): + raise ValueError("wmean: weights are not compatible with the tensor") + + # pyre-fixme[6]: For 1st param expected `Optional[dtype]` but got + # `Union[Tuple[int], int]`. + return (x * weight[..., None]).sum(**args) / weight[..., None].sum(**args).clamp( + eps + ) + + +def eyes( + dim: int, + N: int, + device: Optional[torch.device] = None, + dtype: torch.dtype = torch.float32, +) -> torch.Tensor: + """ + Generates a batch of `N` identity matrices of shape `(N, dim, dim)`. + + Args: + **dim**: The dimensionality of the identity matrices. + **N**: The number of identity matrices. + **device**: The device to be used for allocating the matrices. + **dtype**: The datatype of the matrices. + + Returns: + **identities**: A batch of identity matrices of shape `(N, dim, dim)`. + """ + identities = torch.eye(dim, device=device, dtype=dtype) + return identities[None].repeat(N, 1, 1) + + +def convert_pointclouds_to_tensor(pcl: Union[torch.Tensor, "Pointclouds"]): + """ + If `type(pcl)==Pointclouds`, converts a `pcl` object to a + padded representation and returns it together with the number of points + per batch. Otherwise, returns the input itself with the number of points + set to the size of the second dimension of `pcl`. + """ + if is_pointclouds(pcl): + X = pcl.points_padded() # type: ignore + num_points = pcl.num_points_per_cloud() # type: ignore + elif torch.is_tensor(pcl): + X = pcl + num_points = X.shape[1] * torch.ones( # type: ignore + X.shape[0], + device=X.device, + dtype=torch.int64, + ) + else: + raise ValueError( + "The inputs X, Y should be either Pointclouds objects or tensors." + ) + return X, num_points + + +def is_pointclouds(pcl: Union[torch.Tensor, "Pointclouds"]) -> bool: + """Checks whether the input `pcl` is an instance of `Pointclouds` + by checking the existence of `points_padded` and `num_points_per_cloud` + functions. + """ + return hasattr(pcl, "points_padded") and hasattr(pcl, "num_points_per_cloud") + + +def get_point_covariances( + points_padded: torch.Tensor, + num_points_per_cloud: torch.Tensor, + neighborhood_size: int, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Computes the per-point covariance matrices by of the 3D locations of + K-nearest neighbors of each point. + + Args: + **points_padded**: Input point clouds as a padded tensor + of shape `(minibatch, num_points, dim)`. + **num_points_per_cloud**: Number of points per cloud + of shape `(minibatch,)`. + **neighborhood_size**: Number of nearest neighbors for each point + used to estimate the covariance matrices. + + Returns: + **covariances**: A batch of per-point covariance matrices + of shape `(minibatch, dim, dim)`. + **k_nearest_neighbors**: A batch of `neighborhood_size` nearest + neighbors for each of the point cloud points + of shape `(minibatch, num_points, neighborhood_size, dim)`. + """ + # get K nearest neighbor idx for each point in the point cloud + k_nearest_neighbors = knn_points( + points_padded, + points_padded, + lengths1=num_points_per_cloud, + lengths2=num_points_per_cloud, + K=neighborhood_size, + return_nn=True, + ).knn + # obtain the mean of the neighborhood + pt_mean = k_nearest_neighbors.mean(2, keepdim=True) + # compute the diff of the neighborhood and the mean of the neighborhood + central_diff = k_nearest_neighbors - pt_mean + # per-nn-point covariances + per_pt_cov = central_diff.unsqueeze(4) * central_diff.unsqueeze(3) + # per-point covariances + covariances = per_pt_cov.mean(2) + + return covariances, k_nearest_neighbors diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/vert_align.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/vert_align.py new file mode 100644 index 0000000000000000000000000000000000000000..fd1a90d0f8836c87a279a2e7c1db6be0fa49b3e4 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/ops/vert_align.py @@ -0,0 +1,107 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + + +import torch +import torch.nn.functional as F + + +def vert_align( + feats, + verts, + return_packed: bool = False, + interp_mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: bool = True, +) -> torch.Tensor: + """ + Sample vertex features from a feature map. This operation is called + "perceptual feature pooling" in [1] or "vert align" in [2]. + + [1] Wang et al, "Pixel2Mesh: Generating 3D Mesh Models from Single + RGB Images", ECCV 2018. + [2] Gkioxari et al, "Mesh R-CNN", ICCV 2019 + + Args: + feats: FloatTensor of shape (N, C, H, W) representing image features + from which to sample or a list of features each with potentially + different C, H or W dimensions. + verts: FloatTensor of shape (N, V, 3) or an object (e.g. Meshes or Pointclouds) + with `verts_padded' or `points_padded' as an attribute giving the (x, y, z) + vertex positions for which to sample. (x, y) verts should be normalized such + that (-1, -1) corresponds to top-left and (+1, +1) to bottom-right + location in the input feature map. + return_packed: (bool) Indicates whether to return packed features + interp_mode: (str) Specifies how to interpolate features. + ('bilinear' or 'nearest') + padding_mode: (str) Specifies how to handle vertices outside of the + [-1, 1] range. ('zeros', 'reflection', or 'border') + align_corners (bool): Geometrically, we consider the pixels of the + input as squares rather than points. + If set to ``True``, the extrema (``-1`` and ``1``) are considered as + referring to the center points of the input's corner pixels. If set + to ``False``, they are instead considered as referring to the corner + points of the input's corner pixels, making the sampling more + resolution agnostic. Default: ``True`` + + Returns: + feats_sampled: FloatTensor of shape (N, V, C) giving sampled features for each + vertex. If feats is a list, we return concatenated features in axis=2 of + shape (N, V, sum(C_n)) where C_n = feats[n].shape[1]. + If return_packed = True, the features are transformed to a packed + representation of shape (sum(V), C) + """ + if torch.is_tensor(verts): + if verts.dim() != 3: + raise ValueError("verts tensor should be 3 dimensional") + grid = verts + elif hasattr(verts, "verts_padded"): + grid = verts.verts_padded() + elif hasattr(verts, "points_padded"): + grid = verts.points_padded() + else: + raise ValueError( + "verts must be a tensor or have a " + + "`points_padded' or`verts_padded` attribute." + ) + + grid = grid[:, None, :, :2] # (N, 1, V, 2) + + if torch.is_tensor(feats): + feats = [feats] + for feat in feats: + if feat.dim() != 4: + raise ValueError("feats must have shape (N, C, H, W)") + if grid.shape[0] != feat.shape[0]: + raise ValueError("inconsistent batch dimension") + + feats_sampled = [] + for feat in feats: + feat_sampled = F.grid_sample( + feat, + grid, + mode=interp_mode, + padding_mode=padding_mode, + align_corners=align_corners, + ) # (N, C, 1, V) + feat_sampled = feat_sampled.squeeze(dim=2).transpose(1, 2) # (N, V, C) + feats_sampled.append(feat_sampled) + feats_sampled = torch.cat(feats_sampled, dim=2) # (N, V, sum(C)) + + if return_packed: + # flatten the first two dimensions: (N*V, C) + feats_sampled = feats_sampled.view(-1, feats_sampled.shape[-1]) + if hasattr(verts, "verts_padded_to_packed_idx"): + idx = ( + verts.verts_padded_to_packed_idx() + .view(-1, 1) + .expand(-1, feats_sampled.shape[-1]) + ) + feats_sampled = feats_sampled.gather(0, idx) # (sum(V), C) + + return feats_sampled diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/renderer/implicit/__init__.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/renderer/implicit/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d13affae2db0c661ab1073e50945dd95db5eeba8 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/renderer/implicit/__init__.py @@ -0,0 +1,27 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from .harmonic_embedding import HarmonicEmbedding +from .raymarching import AbsorptionOnlyRaymarcher, EmissionAbsorptionRaymarcher +from .raysampling import ( + GridRaysampler, + MonteCarloRaysampler, + MultinomialRaysampler, + NDCGridRaysampler, + NDCMultinomialRaysampler, +) +from .renderer import ImplicitRenderer, VolumeRenderer, VolumeSampler +from .utils import ( + HeterogeneousRayBundle, + ray_bundle_to_ray_points, + ray_bundle_variables_to_ray_points, + RayBundle, +) + + +__all__ = [k for k in globals().keys() if not k.startswith("_")] diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/renderer/implicit/harmonic_embedding.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/renderer/implicit/harmonic_embedding.py new file mode 100644 index 0000000000000000000000000000000000000000..c3b1e94440c4f6a36643309b42a265ad7a42bcb7 --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/renderer/implicit/harmonic_embedding.py @@ -0,0 +1,184 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from typing import Optional + +import torch + + +class HarmonicEmbedding(torch.nn.Module): + def __init__( + self, + n_harmonic_functions: int = 6, + omega_0: float = 1.0, + logspace: bool = True, + append_input: bool = True, + ) -> None: + """ + The harmonic embedding layer supports the classical + Nerf positional encoding described in + `NeRF `_ + and the integrated position encoding in + `MIP-NeRF `_. + + During the inference you can provide the extra argument `diag_cov`. + + If `diag_cov is None`, it converts + rays parametrized with a `ray_bundle` to 3D points by + extending each ray according to the corresponding length. + Then it converts each feature + (i.e. vector along the last dimension) in `x` + into a series of harmonic features `embedding`, + where for each i in range(dim) the following are present + in embedding[...]:: + + [ + sin(f_1*x[..., i]), + sin(f_2*x[..., i]), + ... + sin(f_N * x[..., i]), + cos(f_1*x[..., i]), + cos(f_2*x[..., i]), + ... + cos(f_N * x[..., i]), + x[..., i], # only present if append_input is True. + ] + + where N corresponds to `n_harmonic_functions-1`, and f_i is a scalar + denoting the i-th frequency of the harmonic embedding. + + + If `diag_cov is not None`, it approximates + conical frustums following a ray bundle as gaussians, + defined by x, the means of the gaussians and diag_cov, + the diagonal covariances. + Then it converts each gaussian + into a series of harmonic features `embedding`, + where for each i in range(dim) the following are present + in embedding[...]:: + + [ + sin(f_1*x[..., i]) * exp(0.5 * f_1**2 * diag_cov[..., i,]), + sin(f_2*x[..., i]) * exp(0.5 * f_2**2 * diag_cov[..., i,]), + ... + sin(f_N * x[..., i]) * exp(0.5 * f_N**2 * diag_cov[..., i,]), + cos(f_1*x[..., i]) * exp(0.5 * f_1**2 * diag_cov[..., i,]), + cos(f_2*x[..., i]) * exp(0.5 * f_2**2 * diag_cov[..., i,]),, + ... + cos(f_N * x[..., i]) * exp(0.5 * f_N**2 * diag_cov[..., i,]), + x[..., i], # only present if append_input is True. + ] + + where N equals `n_harmonic_functions-1`, and f_i is a scalar + denoting the i-th frequency of the harmonic embedding. + + If `logspace==True`, the frequencies `[f_1, ..., f_N]` are + powers of 2: + `f_1, ..., f_N = 2**torch.arange(n_harmonic_functions)` + + If `logspace==False`, frequencies are linearly spaced between + `1.0` and `2**(n_harmonic_functions-1)`: + `f_1, ..., f_N = torch.linspace( + 1.0, 2**(n_harmonic_functions-1), n_harmonic_functions + )` + + Note that `x` is also premultiplied by the base frequency `omega_0` + before evaluating the harmonic functions. + + Args: + n_harmonic_functions: int, number of harmonic + features + omega_0: float, base frequency + logspace: bool, Whether to space the frequencies in + logspace or linear space + append_input: bool, whether to concat the original + input to the harmonic embedding. If true the + output is of the form (embed.sin(), embed.cos(), x) + """ + super().__init__() + + if logspace: + frequencies = 2.0 ** torch.arange( + n_harmonic_functions, + dtype=torch.float32, + ) + else: + frequencies = torch.linspace( + 1.0, + 2.0 ** (n_harmonic_functions - 1), + n_harmonic_functions, + dtype=torch.float32, + ) + + self.register_buffer("_frequencies", frequencies * omega_0, persistent=False) + self.register_buffer( + "_zero_half_pi", torch.tensor([0.0, 0.5 * torch.pi]), persistent=False + ) + self.append_input = append_input + + def forward( + self, x: torch.Tensor, diag_cov: Optional[torch.Tensor] = None, **kwargs + ) -> torch.Tensor: + """ + Args: + x: tensor of shape [..., dim] + diag_cov: An optional tensor of shape `(..., dim)` + representing the diagonal covariance matrices of our Gaussians, joined with x + as means of the Gaussians. + + Returns: + embedding: a harmonic embedding of `x` of shape + [..., (n_harmonic_functions * 2 + int(append_input)) * num_points_per_ray] + """ + # [..., dim, n_harmonic_functions] + embed = x[..., None] * self._frequencies + # [..., 1, dim, n_harmonic_functions] + [2, 1, 1] => [..., 2, dim, n_harmonic_functions] + embed = embed[..., None, :, :] + self._zero_half_pi[..., None, None] + # Use the trig identity cos(x) = sin(x + pi/2) + # and do one vectorized call to sin([x, x+pi/2]) instead of (sin(x), cos(x)). + embed = embed.sin() + if diag_cov is not None: + x_var = diag_cov[..., None] * torch.pow(self._frequencies, 2) + exp_var = torch.exp(-0.5 * x_var) + # [..., 2, dim, n_harmonic_functions] + embed = embed * exp_var[..., None, :, :] + + embed = embed.reshape(*x.shape[:-1], -1) + + if self.append_input: + return torch.cat([embed, x], dim=-1) + return embed + + @staticmethod + def get_output_dim_static( + input_dims: int, + n_harmonic_functions: int, + append_input: bool, + ) -> int: + """ + Utility to help predict the shape of the output of `forward`. + + Args: + input_dims: length of the last dimension of the input tensor + n_harmonic_functions: number of embedding frequencies + append_input: whether or not to concat the original + input to the harmonic embedding + Returns: + int: the length of the last dimension of the output tensor + """ + return input_dims * (2 * n_harmonic_functions + int(append_input)) + + def get_output_dim(self, input_dims: int = 3) -> int: + """ + Same as above. The default for input_dims is 3 for 3D applications + which use harmonic embedding for positional encoding, + so the input might be xyz. + """ + return self.get_output_dim_static( + input_dims, len(self._frequencies), self.append_input + ) diff --git a/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/renderer/implicit/raymarching.py b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/renderer/implicit/raymarching.py new file mode 100644 index 0000000000000000000000000000000000000000..60eae7b765948bfc138fed6e4d8a11b89aa1f54c --- /dev/null +++ b/third_party/AnyBimanual/third_party/pytorch3d/pytorch3d/renderer/implicit/raymarching.py @@ -0,0 +1,233 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +import warnings +from typing import Optional, Tuple, Union + +import torch + + +class EmissionAbsorptionRaymarcher(torch.nn.Module): + """ + Raymarch using the Emission-Absorption (EA) algorithm. + + The algorithm independently renders each ray by analyzing density and + feature values sampled at (typically uniformly) spaced 3D locations along + each ray. The density values `rays_densities` are of shape + `(..., n_points_per_ray)`, their values should range between [0, 1], and + represent the opaqueness of each point (the higher the less transparent). + The feature values `rays_features` of shape + `(..., n_points_per_ray, feature_dim)` represent the content of the + point that is supposed to be rendered in case the given point is opaque + (i.e. its density -> 1.0). + + EA first utilizes `rays_densities` to compute the absorption function + along each ray as follows:: + + absorption = cumprod(1 - rays_densities, dim=-1) + + The value of absorption at position `absorption[..., k]` specifies + how much light has reached `k`-th point along a ray since starting + its trajectory at `k=0`-th point. + + Each ray is then rendered into a tensor `features` of shape `(..., feature_dim)` + by taking a weighed combination of per-ray features `rays_features` as follows:: + + weights = absorption * rays_densities + features = (rays_features * weights).sum(dim=-2) + + Where `weights` denote a function that has a strong peak around the location + of the first surface point that a given ray passes through. + + Note that for a perfectly bounded volume (with a strictly binary density), + the `weights = cumprod(1 - rays_densities, dim=-1) * rays_densities` + function would yield 0 everywhere. In order to prevent this, + the result of the cumulative product is shifted `self.surface_thickness` + elements along the ray direction. + """ + + def __init__(self, surface_thickness: int = 1) -> None: + """ + Args: + surface_thickness: Denotes the overlap between the absorption + function and the density function. + """ + super().__init__() + self.surface_thickness = surface_thickness + + def forward( + self, + rays_densities: torch.Tensor, + rays_features: torch.Tensor, + eps: float = 1e-10, + **kwargs, + ) -> torch.Tensor: + """ + Args: + rays_densities: Per-ray density values represented with a tensor + of shape `(..., n_points_per_ray, 1)` whose values range in [0, 1]. + rays_features: Per-ray feature values represented with a tensor + of shape `(..., n_points_per_ray, feature_dim)`. + eps: A lower bound added to `rays_densities` before computing + the absorption function (cumprod of `1-rays_densities` along + each ray). This prevents the cumprod to yield exact 0 + which would inhibit any gradient-based learning. + + Returns: + features_opacities: A tensor of shape `(..., feature_dim+1)` + that concatenates two tensors along the last dimension: + 1) features: A tensor of per-ray renders + of shape `(..., feature_dim)`. + 2) opacities: A tensor of per-ray opacity values + of shape `(..., 1)`. Its values range between [0, 1] and + denote the total amount of light that has been absorbed + for each ray. E.g. a value of 0 corresponds to the ray + completely passing through a volume. Please refer to the + `AbsorptionOnlyRaymarcher` documentation for the + explanation of the algorithm that computes `opacities`. + """ + _check_raymarcher_inputs( + rays_densities, + rays_features, + None, + z_can_be_none=True, + features_can_be_none=False, + density_1d=True, + ) + _check_density_bounds(rays_densities) + rays_densities = rays_densities[..., 0] + absorption = _shifted_cumprod( + (1.0 + eps) - rays_densities, shift=self.surface_thickness + ) + weights = rays_densities * absorption + features = (weights[..., None] * rays_features).sum(dim=-2) + opacities = 1.0 - torch.prod(1.0 - rays_densities, dim=-1, keepdim=True) + + return torch.cat((features, opacities), dim=-1) + + +class AbsorptionOnlyRaymarcher(torch.nn.Module): + """ + Raymarch using the Absorption-Only (AO) algorithm. + + The algorithm independently renders each ray by analyzing density and + feature values sampled at (typically uniformly) spaced 3D locations along + each ray. The density values `rays_densities` are of shape + `(..., n_points_per_ray, 1)`, their values should range between [0, 1], and + represent the opaqueness of each point (the higher the less transparent). + The algorithm only measures the total amount of light absorbed along each ray + and, besides outputting per-ray `opacity` values of shape `(...,)`, + does not produce any feature renderings. + + The algorithm simply computes `total_transmission = prod(1 - rays_densities)` + of shape `(..., 1)` which, for each ray, measures the total amount of light + that passed through the volume. + It then returns `opacities = 1 - total_transmission`. + """ + + def __init__(self) -> None: + super().__init__() + + def forward( + self, rays_densities: torch.Tensor, **kwargs + ) -> Union[None, torch.Tensor]: + """ + Args: + rays_densities: Per-ray density values represented with a tensor + of shape `(..., n_points_per_ray)` whose values range in [0, 1]. + + Returns: + opacities: A tensor of per-ray opacity values of shape `(..., 1)`. + Its values range between [0, 1] and denote the total amount + of light that has been absorbed for each ray. E.g. a value + of 0 corresponds to the ray completely passing through a volume. + """ + + _check_raymarcher_inputs( + rays_densities, + None, + None, + features_can_be_none=True, + z_can_be_none=True, + density_1d=True, + ) + rays_densities = rays_densities[..., 0] + _check_density_bounds(rays_densities) + total_transmission = torch.prod(1 - rays_densities, dim=-1, keepdim=True) + opacities = 1.0 - total_transmission + return opacities + + +def _shifted_cumprod(x, shift: int = 1): + """ + Computes `torch.cumprod(x, dim=-1)` and prepends `shift` number of + ones and removes `shift` trailing elements to/from the last dimension + of the result. + """ + x_cumprod = torch.cumprod(x, dim=-1) + x_cumprod_shift = torch.cat( + [torch.ones_like(x_cumprod[..., :shift]), x_cumprod[..., :-shift]], dim=-1 + ) + return x_cumprod_shift + + +def _check_density_bounds( + rays_densities: torch.Tensor, bounds: Tuple[float, float] = (0.0, 1.0) +) -> None: + """ + Checks whether the elements of `rays_densities` range within `bounds`. + If not issues a warning. + """ + with torch.no_grad(): + if (rays_densities.max() > bounds[1]) or (rays_densities.min() < bounds[0]): + warnings.warn( + "One or more elements of rays_densities are outside of valid" + + f"range {str(bounds)}" + ) + + +def _check_raymarcher_inputs( + rays_densities: torch.Tensor, + rays_features: Optional[torch.Tensor], + rays_z: Optional[torch.Tensor], + features_can_be_none: bool = False, + z_can_be_none: bool = False, + density_1d: bool = True, +) -> None: + """ + Checks the validity of the inputs to raymarching algorithms. + """ + if not torch.is_tensor(rays_densities): + raise ValueError("rays_densities has to be an instance of torch.Tensor.") + + if not z_can_be_none and not torch.is_tensor(rays_z): + raise ValueError("rays_z has to be an instance of torch.Tensor.") + + if not features_can_be_none and not torch.is_tensor(rays_features): + raise ValueError("rays_features has to be an instance of torch.Tensor.") + + if rays_densities.ndim < 1: + raise ValueError("rays_densities have to have at least one dimension.") + + if density_1d and rays_densities.shape[-1] != 1: + raise ValueError( + "The size of the last dimension of rays_densities has to be one." + + f" Got shape {rays_densities.shape}." + ) + + rays_shape = rays_densities.shape[:-1] + + # pyre-fixme[16]: `Optional` has no attribute `shape`. + if not z_can_be_none and rays_z.shape != rays_shape: + raise ValueError("rays_z have to be of the same shape as rays_densities.") + + if not features_can_be_none and rays_features.shape[:-1] != rays_shape: + raise ValueError( + "The first to previous to last dimensions of rays_features" + " have to be the same as all dimensions of rays_densities." + )