MinkowskiEngine/tests/python/tensor_field.py

# Copyright (c) 2020 NVIDIA CORPORATION.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# Please cite "4D Spatio-Temporal ConvNets: Minkowski Convolutional Neural
# Networks", CVPR'19 (https://arxiv.org/abs/1904.08755) if you use any part
# of the code.
import torch
import unittest
import torch.nn as nn

from tests.python.common import load_file
from MinkowskiEngine.utils import batched_coordinates, sparse_quantize
from MinkowskiTensor import SparseTensorQuantizationMode
from MinkowskiTensorField import TensorField
from MinkowskiOps import MinkowskiLinear, MinkowskiToSparseTensor
from MinkowskiNonlinearity import MinkowskiReLU
from MinkowskiNormalization import MinkowskiBatchNorm
from MinkowskiConvolution import MinkowskiConvolution, MinkowskiConvolutionTranspose


class TestTensorField(unittest.TestCase):
    def test(self):
        coords = torch.IntTensor(
            [[0, 1], [0, 1], [0, 2], [0, 2], [1, 0], [1, 0], [1, 1]]
        )
        feats = torch.FloatTensor([[0, 1, 2, 3, 5, 6, 7]]).T
        sfield = TensorField(feats, coords)

        # Convert to a sparse tensor
        stensor = sfield.sparse(
            quantization_mode=SparseTensorQuantizationMode.UNWEIGHTED_AVERAGE
        )
        print(stensor)
        self.assertTrue(
            {0.5, 2.5, 5.5, 7} == {a for a in stensor.F.squeeze().detach().numpy()}
        )

        # device cuda
        if not torch.cuda.is_available():
            return

        sfield = TensorField(feats, coords, device="cuda")

        # Convert to a sparse tensor
        stensor = sfield.sparse(
            quantization_mode=SparseTensorQuantizationMode.UNWEIGHTED_AVERAGE
        )
        print(stensor)
        self.assertTrue(
            {0.5, 2.5, 5.5, 7}
            == {a for a in stensor.F.squeeze().detach().cpu().numpy()}
        )

    def test_maxpool(self):
        coords = torch.IntTensor(
            [[0, 1], [0, 1], [0, 2], [0, 2], [1, 0], [1, 0], [1, 1]]
        )
        feats = torch.FloatTensor([[0, 1, 2, 3, 5, 6, 7]]).T
        sfield = TensorField(feats, coords)

        # Convert to a sparse tensor
        stensor = sfield.sparse(quantization_mode=SparseTensorQuantizationMode.MAX_POOL)
        print(stensor)
        self.assertTrue(
            {1, 3, 6, 7} == {a for a in stensor.F.squeeze().detach().numpy()}
        )

        # device cuda
        if not torch.cuda.is_available():
            return

        sfield = TensorField(feats, coords, device="cuda")

        # Convert to a sparse tensor
        stensor = sfield.sparse(quantization_mode=SparseTensorQuantizationMode.MAX_POOL)
        print(stensor)
        self.assertTrue(
            {1, 3, 6, 7} == {a for a in stensor.F.squeeze().detach().cpu().numpy()}
        )

    def test_pcd(self):
        coords, colors, pcd = load_file("1.ply")
        voxel_size = 0.02
        colors = torch.from_numpy(colors)
        bcoords = batched_coordinates([coords / voxel_size])
        tfield = TensorField(colors, bcoords)

        self.assertTrue(len(tfield) == len(colors))
        stensor = tfield.sparse()
        print(stensor)

    def test_network(self):
        coords, colors, pcd = load_file("1.ply")
        voxel_size = 0.02
        colors = torch.from_numpy(colors)
        bcoords = batched_coordinates([coords / voxel_size])
        tfield = TensorField(colors, bcoords).float()

        network = nn.Sequential(
            MinkowskiLinear(3, 16),
            MinkowskiBatchNorm(16),
            MinkowskiReLU(),
            MinkowskiLinear(16, 32),
            MinkowskiBatchNorm(32),
            MinkowskiReLU(),
            MinkowskiToSparseTensor(),
            MinkowskiConvolution(32, 64, kernel_size=3, stride=2, dimension=3),
        )

        print(network(tfield))

    def test_network_device(self):
        coords, colors, pcd = load_file("1.ply")
        voxel_size = 0.02
        colors = torch.from_numpy(colors)
        bcoords = batched_coordinates([coords / voxel_size])
        tfield = TensorField(colors, bcoords, device=0).float()

        network = nn.Sequential(
            MinkowskiLinear(3, 16),
            MinkowskiBatchNorm(16),
            MinkowskiReLU(),
            MinkowskiLinear(16, 32),
            MinkowskiBatchNorm(32),
            MinkowskiReLU(),
            MinkowskiToSparseTensor(),
            MinkowskiConvolution(32, 64, kernel_size=3, stride=2, dimension=3),
        ).to(0)

        print(network(tfield))

    def slice(self):
        device = "cuda"
        coords, colors, pcd = load_file("1.ply")
        voxel_size = 0.02
        colors = torch.from_numpy(colors).float()
        bcoords = batched_coordinates([coords / voxel_size], dtype=torch.float32)
        tfield = TensorField(colors, bcoords, device=device)

        network = nn.Sequential(
            MinkowskiLinear(3, 16),
            MinkowskiBatchNorm(16),
            MinkowskiReLU(),
            MinkowskiLinear(16, 32),
            MinkowskiBatchNorm(32),
            MinkowskiReLU(),
            MinkowskiToSparseTensor(),
            MinkowskiConvolution(32, 64, kernel_size=3, stride=2, dimension=3),
            MinkowskiConvolutionTranspose(64, 32, kernel_size=3, stride=2, dimension=3),
        ).to(device)

        otensor = network(tfield)
        ofield = otensor.slice(tfield)
        self.assertEqual(len(tfield), len(ofield))
        self.assertEqual(ofield.F.size(1), otensor.F.size(1))
        ofield = otensor.cat_slice(tfield)
        self.assertEqual(len(tfield), len(ofield))
        self.assertEqual(ofield.F.size(1), (otensor.F.size(1) + tfield.F.size(1)))

    def slice_no_duplicate(self):
        coords, colors, pcd = load_file("1.ply")
        voxel_size = 0.02
        # Extract unique coords
        coords, colors = sparse_quantize(coords / voxel_size, colors)
        bcoords = batched_coordinates([coords], dtype=torch.float32)
        colors = torch.from_numpy(colors).float()
        tfield = TensorField(colors, bcoords)

        network = nn.Sequential(
            MinkowskiLinear(3, 16),
            MinkowskiBatchNorm(16),
            MinkowskiReLU(),
            MinkowskiLinear(16, 32),
            MinkowskiBatchNorm(32),
            MinkowskiReLU(),
            MinkowskiToSparseTensor(),
            MinkowskiConvolution(32, 64, kernel_size=3, stride=2, dimension=3),
            MinkowskiConvolutionTranspose(64, 32, kernel_size=3, stride=2, dimension=3),
        )

        otensor = network(tfield)
        ofield = otensor.slice(tfield)
        self.assertEqual(len(tfield), len(ofield))
        self.assertEqual(ofield.F.size(1), otensor.F.size(1))
        ofield = otensor.cat_slice(tfield)
        self.assertEqual(len(tfield), len(ofield))
        self.assertEqual(ofield.F.size(1), (otensor.F.size(1) + tfield.F.size(1)))

    def stride_slice(self):
        coords, colors, pcd = load_file("1.ply")
        voxel_size = 0.02
        colors = torch.from_numpy(colors).float()
        bcoords = batched_coordinates([coords / voxel_size], dtype=torch.float32)
        tfield = TensorField(colors, bcoords)

        network = nn.Sequential(
            MinkowskiToSparseTensor(),
            MinkowskiConvolution(3, 8, kernel_size=3, stride=4, dimension=3),
            MinkowskiReLU(),
            MinkowskiConvolution(8, 16, kernel_size=3, stride=4, dimension=3),
        )

        otensor = network(tfield)
        ofield = otensor.slice(tfield)
        self.assertTrue(len(ofield) == len(tfield))

    def field_to_sparse(self):
        coords, colors, pcd = load_file("1.ply")
        voxel_size = 0.02
        colors = torch.from_numpy(colors).float()
        bcoords = batched_coordinates([coords / voxel_size], dtype=torch.float32)
        tfield = TensorField(colors, bcoords)

        network = nn.Sequential(
            MinkowskiToSparseTensor(),
            MinkowskiConvolution(3, 8, kernel_size=3, stride=4, dimension=3),
            MinkowskiReLU(),
            MinkowskiConvolution(8, 16, kernel_size=3, stride=4, dimension=3),
        )

        otensor = network(tfield)
        otensor.F.sum().backward()
        field_to_sparse = tfield.sparse(coordinate_map_key=otensor.coordinate_map_key)
        self.assertTrue(len(field_to_sparse.F) == len(otensor))


class TestTensorFieldSplat(unittest.TestCase):
    def setUp(self):
        coords, colors, pcd = load_file("1.ply")
        voxel_size = 0.02
        colors = torch.from_numpy(colors).float()
        bcoords = batched_coordinates([coords / voxel_size], dtype=torch.float32)
        self.tensor_field = TensorField(coordinates=bcoords, features=colors)

    def test_splat(self):
        self.tensor_field.splat()

    def test_small(self):
        coords = torch.FloatTensor([[0, 0.1], [0, 1.1]])
        feats = torch.FloatTensor([[1], [2]])
        tfield = TensorField(coordinates=coords, features=feats)
        tensor = tfield.splat()
        print(tfield)
        print(tensor)
        print(tensor.interpolate(tfield))

    def test_small2(self):
        coords = torch.FloatTensor([[0, 0.1, 0.1], [0, 1.1, 1.1]])
        feats = torch.FloatTensor([[1], [2]])
        tfield = TensorField(coordinates=coords, features=feats)
        tensor = tfield.splat()
        print(tfield)
        print(tensor)
        print(tensor.interpolate(tfield))