Add files using upload-large-folder tool

.gitattributes

@@ -336,3 +336,6 @@ my_container_sandbox/workspace/anaconda3/lib/python3.8/site-packages/scipy.libs/
 my_container_sandbox/workspace/anaconda3/lib/python3.8/site-packages/brotli/_brotli.abi3.so filter=lfs diff=lfs merge=lfs -text
 my_container_sandbox/workspace/anaconda3/lib/python3.8/site-packages/opencv_python.libs/libxcb-xkb-9ba31ab3.so.1.0.0 filter=lfs diff=lfs merge=lfs -text
 my_container_sandbox/workspace/anaconda3/lib/python3.8/site-packages/opencv_python.libs/libssl-28bef1ac.so.1.1 filter=lfs diff=lfs merge=lfs -text
+my_container_sandbox/workspace/anaconda3/lib/python3.8/site-packages/opencv_python.libs/libxkbcommon-71ae2972.so.0.0.0 filter=lfs diff=lfs merge=lfs -text
+my_container_sandbox/workspace/anaconda3/lib/libnvvm.so.4.0.0 filter=lfs diff=lfs merge=lfs -text
+my_container_sandbox/workspace/anaconda3/lib/python3.8/site-packages/sklearn/_isotonic.cpython-38-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text

my_container_sandbox/workspace/anaconda3/lib/libnvvm.so.4.0.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0c95d249f3d60d67dbd191174fe1d8b9f393b1887ae1a54b4988823c7e42a31
+size 26650200

my_container_sandbox/workspace/anaconda3/lib/python3.8/site-packages/monai/networks/blocks/__init__.py

@@ -0,0 +1,40 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .acti_norm import ADN
+from .activation import MemoryEfficientSwish, Mish, Swish
+from .aspp import SimpleASPP
+from .backbone_fpn_utils import BackboneWithFPN
+from .convolutions import Convolution, ResidualUnit
+from .crf import CRF
+from .dints_block import ActiConvNormBlock, FactorizedIncreaseBlock, FactorizedReduceBlock, P3DActiConvNormBlock
+from .downsample import MaxAvgPool
+from .dynunet_block import UnetBasicBlock, UnetOutBlock, UnetResBlock, UnetUpBlock, get_output_padding, get_padding
+from .fcn import FCN, GCN, MCFCN, Refine
+from .feature_pyramid_network import ExtraFPNBlock, FeaturePyramidNetwork, LastLevelMaxPool, LastLevelP6P7
+from .localnet_block import LocalNetDownSampleBlock, LocalNetFeatureExtractorBlock, LocalNetUpSampleBlock
+from .mlp import MLPBlock
+from .patchembedding import PatchEmbed, PatchEmbeddingBlock
+from .regunet_block import RegistrationDownSampleBlock, RegistrationExtractionBlock, RegistrationResidualConvBlock
+from .segresnet_block import ResBlock
+from .selfattention import SABlock
+from .squeeze_and_excitation import (
+    ChannelSELayer,
+    ResidualSELayer,
+    SEBlock,
+    SEBottleneck,
+    SEResNetBottleneck,
+    SEResNeXtBottleneck,
+)
+from .transformerblock import TransformerBlock
+from .unetr_block import UnetrBasicBlock, UnetrPrUpBlock, UnetrUpBlock
+from .upsample import SubpixelUpsample, Subpixelupsample, SubpixelUpSample, Upsample, UpSample
+from .warp import DVF2DDF, Warp

my_container_sandbox/workspace/anaconda3/lib/python3.8/site-packages/monai/networks/blocks/acti_norm.py

@@ -0,0 +1,101 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional, Tuple, Union
+
+import torch.nn as nn
+
+from monai.networks.layers.utils import get_act_layer, get_dropout_layer, get_norm_layer
+
+
+class ADN(nn.Sequential):
+    """
+    Constructs a sequential module of optional activation (A), dropout (D), and normalization (N) layers
+    with an arbitrary order::
+
+        -- (Norm) -- (Dropout) -- (Acti) --
+
+    Args:
+        ordering: a string representing the ordering of activation, dropout, and normalization. Defaults to "NDA".
+        in_channels: `C` from an expected input of size (N, C, H[, W, D]).
+        act: activation type and arguments. Defaults to PReLU.
+        norm: feature normalization type and arguments. Defaults to instance norm.
+        norm_dim: determine the spatial dimensions of the normalization layer.
+            defaults to `dropout_dim` if unspecified.
+        dropout: dropout ratio. Defaults to no dropout.
+        dropout_dim: determine the spatial dimensions of dropout.
+            defaults to `norm_dim` if unspecified.
+
+            - When dropout_dim = 1, randomly zeroes some of the elements for each channel.
+            - When dropout_dim = 2, Randomly zeroes out entire channels (a channel is a 2D feature map).
+            - When dropout_dim = 3, Randomly zeroes out entire channels (a channel is a 3D feature map).
+
+    Examples::
+
+        # activation, group norm, dropout
+        >>> norm_params = ("GROUP", {"num_groups": 1, "affine": False})
+        >>> ADN(norm=norm_params, in_channels=1, dropout_dim=1, dropout=0.8, ordering="AND")
+        ADN(
+            (A): ReLU()
+            (N): GroupNorm(1, 1, eps=1e-05, affine=False)
+            (D): Dropout(p=0.8, inplace=False)
+        )
+
+        # LeakyReLU, dropout
+        >>> act_params = ("leakyrelu", {"negative_slope": 0.1, "inplace": True})
+        >>> ADN(act=act_params, in_channels=1, dropout_dim=1, dropout=0.8, ordering="AD")
+        ADN(
+            (A): LeakyReLU(negative_slope=0.1, inplace=True)
+            (D): Dropout(p=0.8, inplace=False)
+        )
+
+    See also:
+
+        :py:class:`monai.networks.layers.Dropout`
+        :py:class:`monai.networks.layers.Act`
+        :py:class:`monai.networks.layers.Norm`
+        :py:class:`monai.networks.layers.split_args`
+
+    """
+
+    def __init__(
+        self,
+        ordering: str = "NDA",
+        in_channels: Optional[int] = None,
+        act: Optional[Union[Tuple, str]] = "RELU",
+        norm: Optional[Union[Tuple, str]] = None,
+        norm_dim: Optional[int] = None,
+        dropout: Optional[Union[Tuple, str, float]] = None,
+        dropout_dim: Optional[int] = None,
+    ) -> None:
+        super().__init__()
+
+        op_dict = {"A": None, "D": None, "N": None}
+        # define the normalization type and the arguments to the constructor
+        if norm is not None:
+            if norm_dim is None and dropout_dim is None:
+                raise ValueError("norm_dim or dropout_dim needs to be specified.")
+            op_dict["N"] = get_norm_layer(name=norm, spatial_dims=norm_dim or dropout_dim, channels=in_channels)
+
+        # define the activation type and the arguments to the constructor
+        if act is not None:
+            op_dict["A"] = get_act_layer(act)
+
+        if dropout is not None:
+            if norm_dim is None and dropout_dim is None:
+                raise ValueError("norm_dim or dropout_dim needs to be specified.")
+            op_dict["D"] = get_dropout_layer(name=dropout, dropout_dim=dropout_dim or norm_dim)
+
+        for item in ordering.upper():
+            if item not in op_dict:
+                raise ValueError(f"ordering must be a string of {op_dict}, got {item} in it.")
+            if op_dict[item] is not None:
+                self.add_module(item, op_dict[item])

my_container_sandbox/workspace/anaconda3/lib/python3.8/site-packages/monai/networks/blocks/activation.py

@@ -0,0 +1,162 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+from torch import nn
+
+from monai.utils import optional_import
+
+if optional_import("torch.nn.functional", name="mish")[1]:
+
+    def monai_mish(x, inplace: bool = False):
+        return torch.nn.functional.mish(x, inplace=inplace)
+
+else:
+
+    def monai_mish(x, inplace: bool = False):
+        return x * torch.tanh(torch.nn.functional.softplus(x))
+
+
+if optional_import("torch.nn.functional", name="silu")[1]:
+
+    def monai_swish(x, inplace: bool = False):
+        return torch.nn.functional.silu(x, inplace=inplace)
+
+else:
+
+    def monai_swish(x, inplace: bool = False):
+        return SwishImplementation.apply(x)
+
+
+class Swish(nn.Module):
+    r"""Applies the element-wise function:
+
+    .. math::
+        \text{Swish}(x) = x * \text{Sigmoid}(\alpha * x) ~~~~\text{for constant value}~ \alpha.
+
+    Citation: Searching for Activation Functions, Ramachandran et al., 2017, https://arxiv.org/abs/1710.05941.
+
+
+    Shape:
+        - Input: :math:`(N, *)` where `*` means, any number of additional dimensions
+        - Output: :math:`(N, *)`, same shape as the input
+
+
+    Examples::
+
+        >>> import torch
+        >>> from monai.networks.layers.factories import Act
+        >>> m = Act['swish']()
+        >>> input = torch.randn(2)
+        >>> output = m(input)
+    """
+
+    def __init__(self, alpha=1.0):
+        super().__init__()
+        self.alpha = alpha
+
+    def forward(self, input: torch.Tensor) -> torch.Tensor:
+        return input * torch.sigmoid(self.alpha * input)
+
+
+class SwishImplementation(torch.autograd.Function):
+    r"""Memory efficient implementation for training
+    Follows recommendation from:
+    https://github.com/lukemelas/EfficientNet-PyTorch/issues/18#issuecomment-511677853
+
+    Results in ~ 30% memory saving during training as compared to Swish()
+    """
+
+    @staticmethod
+    def forward(ctx, input):
+        result = input * torch.sigmoid(input)
+        ctx.save_for_backward(input)
+        return result
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        input = ctx.saved_tensors[0]
+        sigmoid_input = torch.sigmoid(input)
+        return grad_output * (sigmoid_input * (1 + input * (1 - sigmoid_input)))
+
+
+class MemoryEfficientSwish(nn.Module):
+    r"""Applies the element-wise function:
+
+    .. math::
+        \text{Swish}(x) = x * \text{Sigmoid}(\alpha * x) ~~~~\text{for constant value}~ \alpha=1.
+
+    Memory efficient implementation for training following recommendation from:
+    https://github.com/lukemelas/EfficientNet-PyTorch/issues/18#issuecomment-511677853
+
+    Results in ~ 30% memory saving during training as compared to Swish()
+
+    Citation: Searching for Activation Functions, Ramachandran et al., 2017, https://arxiv.org/abs/1710.05941.
+
+    From Pytorch 1.7.0+, the optimized version of `Swish` named `SiLU` is implemented,
+    this class will utilize `torch.nn.functional.silu` to do the calculation if meets the version.
+
+    Shape:
+        - Input: :math:`(N, *)` where `*` means, any number of additional
+          dimensions
+        - Output: :math:`(N, *)`, same shape as the input
+
+
+    Examples::
+
+        >>> import torch
+        >>> from monai.networks.layers.factories import Act
+        >>> m = Act['memswish']()
+        >>> input = torch.randn(2)
+        >>> output = m(input)
+    """
+
+    def __init__(self, inplace: bool = False):
+        super().__init__()
+        # inplace only works when using torch.nn.functional.silu
+        self.inplace = inplace
+
+    def forward(self, input: torch.Tensor):
+        return monai_swish(input, self.inplace)
+
+
+class Mish(nn.Module):
+    r"""Applies the element-wise function:
+
+    .. math::
+        \text{Mish}(x) = x * tanh(\text{softplus}(x)).
+
+    Citation: Mish: A Self Regularized Non-Monotonic Activation Function, Diganta Misra, 2019, https://arxiv.org/abs/1908.08681.
+
+    From Pytorch 1.9.0+, the optimized version of `Mish` is implemented,
+    this class will utilize `torch.nn.functional.mish` to do the calculation if meets the version.
+
+    Shape:
+        - Input: :math:`(N, *)` where `*` means, any number of additional dimensions
+        - Output: :math:`(N, *)`, same shape as the input
+
+
+    Examples::
+
+        >>> import torch
+        >>> from monai.networks.layers.factories import Act
+        >>> m = Act['mish']()
+        >>> input = torch.randn(2)
+        >>> output = m(input)
+    """
+
+    def __init__(self, inplace: bool = False):
+        super().__init__()
+        # inplace only works when using torch.nn.functional.mish
+        self.inplace = inplace
+
+    def forward(self, input: torch.Tensor):
+        return monai_mish(input, self.inplace)

my_container_sandbox/workspace/anaconda3/lib/python3.8/site-packages/monai/networks/blocks/aspp.py

@@ -0,0 +1,105 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional, Sequence, Tuple, Union
+
+import torch
+import torch.nn as nn
+
+from monai.networks.blocks.convolutions import Convolution
+from monai.networks.layers import same_padding
+from monai.networks.layers.factories import Conv
+
+
+class SimpleASPP(nn.Module):
+    """
+    A simplified version of the atrous spatial pyramid pooling (ASPP) module.
+
+    Chen et al., Encoder-Decoder with Atrous Separable Convolution for Semantic Image Segmentation.
+    https://arxiv.org/abs/1802.02611
+
+    Wang et al., A Noise-robust Framework for Automatic Segmentation of COVID-19 Pneumonia Lesions
+    from CT Images. https://ieeexplore.ieee.org/document/9109297
+    """
+
+    def __init__(
+        self,
+        spatial_dims: int,
+        in_channels: int,
+        conv_out_channels: int,
+        kernel_sizes: Sequence[int] = (1, 3, 3, 3),
+        dilations: Sequence[int] = (1, 2, 4, 6),
+        norm_type: Optional[Union[Tuple, str]] = "BATCH",
+        acti_type: Optional[Union[Tuple, str]] = "LEAKYRELU",
+        bias: bool = False,
+    ) -> None:
+        """
+        Args:
+            spatial_dims: number of spatial dimensions, could be 1, 2, or 3.
+            in_channels: number of input channels.
+            conv_out_channels: number of output channels of each atrous conv.
+                The final number of output channels is conv_out_channels * len(kernel_sizes).
+            kernel_sizes: a sequence of four convolutional kernel sizes.
+                Defaults to (1, 3, 3, 3) for four (dilated) convolutions.
+            dilations: a sequence of four convolutional dilation parameters.
+                Defaults to (1, 2, 4, 6) for four (dilated) convolutions.
+            norm_type: final kernel-size-one convolution normalization type.
+                Defaults to batch norm.
+            acti_type: final kernel-size-one convolution activation type.
+                Defaults to leaky ReLU.
+            bias: whether to have a bias term in convolution blocks. Defaults to False.
+                According to `Performance Tuning Guide <https://pytorch.org/tutorials/recipes/recipes/tuning_guide.html>`_,
+                if a conv layer is directly followed by a batch norm layer, bias should be False.
+
+        Raises:
+            ValueError: When ``kernel_sizes`` length differs from ``dilations``.
+
+        See also:
+
+            :py:class:`monai.networks.layers.Act`
+            :py:class:`monai.networks.layers.Conv`
+            :py:class:`monai.networks.layers.Norm`
+
+        """
+        super().__init__()
+        if len(kernel_sizes) != len(dilations):
+            raise ValueError(
+                "kernel_sizes and dilations length must match, "
+                f"got kernel_sizes={len(kernel_sizes)} dilations={len(dilations)}."
+            )
+        pads = tuple(same_padding(k, d) for k, d in zip(kernel_sizes, dilations))
+
+        self.convs = nn.ModuleList()
+        for k, d, p in zip(kernel_sizes, dilations, pads):
+            _conv = Conv[Conv.CONV, spatial_dims](
+                in_channels=in_channels, out_channels=conv_out_channels, kernel_size=k, dilation=d, padding=p
+            )
+            self.convs.append(_conv)
+
+        out_channels = conv_out_channels * len(pads)  # final conv. output channels
+        self.conv_k1 = Convolution(
+            spatial_dims=spatial_dims,
+            in_channels=out_channels,
+            out_channels=out_channels,
+            kernel_size=1,
+            act=acti_type,
+            norm=norm_type,
+            bias=bias,
+        )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        """
+        Args:
+            x: in shape (batch, channel, spatial_1[, spatial_2, ...]).
+        """
+        x_out = torch.cat([conv(x) for conv in self.convs], dim=1)
+        x_out = self.conv_k1(x_out)
+        return x_out

my_container_sandbox/workspace/anaconda3/lib/python3.8/site-packages/monai/networks/blocks/backbone_fpn_utils.py

@@ -0,0 +1,176 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# =========================================================================
+# Adapted from https://github.com/pytorch/vision/blob/release/0.12/torchvision/models/detection/backbone_utils.py
+# which has the following license...
+# https://github.com/pytorch/vision/blob/main/LICENSE
+#
+# BSD 3-Clause License
+
+# Copyright (c) Soumith Chintala 2016,
+# All rights reserved.
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""
+This script is modified from from torchvision to support N-D images,
+by overriding the definition of convolutional layers and pooling layers.
+
+https://github.com/pytorch/vision/blob/release/0.12/torchvision/models/detection/backbone_utils.py
+"""
+
+from typing import Dict, List, Optional, Union
+
+from torch import Tensor, nn
+
+from monai.networks.nets import resnet
+from monai.utils import optional_import
+
+from .feature_pyramid_network import ExtraFPNBlock, FeaturePyramidNetwork, LastLevelMaxPool
+
+torchvision_models, _ = optional_import("torchvision.models")
+
+__all__ = ["BackboneWithFPN"]
+
+
+class BackboneWithFPN(nn.Module):
+    """
+    Adds an FPN on top of a model.
+    Internally, it uses torchvision.models._utils.IntermediateLayerGetter to
+    extract a submodel that returns the feature maps specified in return_layers.
+    The same limitations of IntermediateLayerGetter apply here.
+
+    Same code as https://github.com/pytorch/vision/blob/release/0.12/torchvision/models/detection/backbone_utils.py
+    Except that this class uses spatial_dims
+
+    Args:
+        backbone: backbone network
+        return_layers: a dict containing the names
+            of the modules for which the activations will be returned as
+            the key of the dict, and the value of the dict is the name
+            of the returned activation (which the user can specify).
+        in_channels_list: number of channels for each feature map
+            that is returned, in the order they are present in the OrderedDict
+        out_channels: number of channels in the FPN.
+        spatial_dims: 2D or 3D images
+    """
+
+    def __init__(
+        self,
+        backbone: nn.Module,
+        return_layers: Dict[str, str],
+        in_channels_list: List[int],
+        out_channels: int,
+        spatial_dims: Union[int, None] = None,
+        extra_blocks: Optional[ExtraFPNBlock] = None,
+    ) -> None:
+        super().__init__()
+
+        # if spatial_dims is not specified, try to find it from backbone.
+        if spatial_dims is None:
+            if hasattr(backbone, "spatial_dims") and isinstance(backbone.spatial_dims, int):
+                spatial_dims = backbone.spatial_dims
+            elif isinstance(backbone.conv1, nn.Conv2d):
+                spatial_dims = 2
+            elif isinstance(backbone.conv1, nn.Conv3d):
+                spatial_dims = 3
+            else:
+                raise ValueError("Could not find spatial_dims of backbone, please specify it.")
+
+        if extra_blocks is None:
+            extra_blocks = LastLevelMaxPool(spatial_dims)
+
+        self.body = torchvision_models._utils.IntermediateLayerGetter(backbone, return_layers=return_layers)
+        self.fpn = FeaturePyramidNetwork(
+            spatial_dims=spatial_dims,
+            in_channels_list=in_channels_list,
+            out_channels=out_channels,
+            extra_blocks=extra_blocks,
+        )
+        self.out_channels = out_channels
+
+    def forward(self, x: Tensor) -> Dict[str, Tensor]:
+        """
+        Computes the resulted feature maps of the network.
+
+        Args:
+            x: input images
+
+        Returns:
+            feature maps after FPN layers. They are ordered from highest resolution first.
+        """
+        x = self.body(x)  # backbone
+        y: Dict[str, Tensor] = self.fpn(x)  # FPN
+        return y
+
+
+def _resnet_fpn_extractor(
+    backbone: resnet.ResNet,
+    spatial_dims: int,
+    trainable_layers: int = 5,
+    returned_layers: Optional[List[int]] = None,
+    extra_blocks: Optional[ExtraFPNBlock] = None,
+) -> BackboneWithFPN:
+    """
+    Same code as https://github.com/pytorch/vision/blob/release/0.12/torchvision/models/detection/backbone_utils.py
+    Except that ``in_channels_stage2 = backbone.in_planes // 8`` instead of ``in_channels_stage2 = backbone.inplanes // 8``,
+    and it requires spatial_dims: 2D or 3D images.
+    """
+
+    # select layers that wont be frozen
+    if trainable_layers < 0 or trainable_layers > 5:
+        raise ValueError(f"Trainable layers should be in the range [0,5], got {trainable_layers}")
+    layers_to_train = ["layer4", "layer3", "layer2", "layer1", "conv1"][:trainable_layers]
+    if trainable_layers == 5:
+        layers_to_train.append("bn1")
+    for name, parameter in backbone.named_parameters():
+        if all([not name.startswith(layer) for layer in layers_to_train]):
+            parameter.requires_grad_(False)
+
+    if extra_blocks is None:
+        extra_blocks = LastLevelMaxPool(spatial_dims)
+
+    if returned_layers is None:
+        returned_layers = [1, 2, 3, 4]
+    if min(returned_layers) <= 0 or max(returned_layers) >= 5:
+        raise ValueError(f"Each returned layer should be in the range [1,4]. Got {returned_layers}")
+    return_layers = {f"layer{k}": str(v) for v, k in enumerate(returned_layers)}
+
+    in_channels_stage2 = backbone.in_planes // 8
+    in_channels_list = [in_channels_stage2 * 2 ** (i - 1) for i in returned_layers]
+    out_channels = 256
+    return BackboneWithFPN(
+        backbone, return_layers, in_channels_list, out_channels, extra_blocks=extra_blocks, spatial_dims=spatial_dims
+    )

my_container_sandbox/workspace/anaconda3/lib/python3.8/site-packages/monai/networks/blocks/convolutions.py

@@ -0,0 +1,326 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional, Sequence, Tuple, Union
+
+import numpy as np
+import torch
+import torch.nn as nn
+
+from monai.networks.blocks import ADN
+from monai.networks.layers.convutils import same_padding, stride_minus_kernel_padding
+from monai.networks.layers.factories import Conv
+from monai.utils.deprecate_utils import deprecated_arg
+
+
+class Convolution(nn.Sequential):
+    """
+    Constructs a convolution with normalization, optional dropout, and optional activation layers::
+
+        -- (Conv|ConvTrans) -- (Norm -- Dropout -- Acti) --
+
+    if ``conv_only`` set to ``True``::
+
+        -- (Conv|ConvTrans) --
+
+    For example:
+
+    .. code-block:: python
+
+        from monai.networks.blocks import Convolution
+
+        conv = Convolution(
+            dimensions=3,
+            in_channels=1,
+            out_channels=1,
+            adn_ordering="ADN",
+            act=("prelu", {"init": 0.2}),
+            dropout=0.1,
+            norm=("layer", {"normalized_shape": (10, 10, 10)}),
+        )
+        print(conv)
+
+    output::
+
+        Convolution(
+          (conv): Conv3d(1, 1, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+          (adn): ADN(
+            (A): PReLU(num_parameters=1)
+            (D): Dropout(p=0.1, inplace=False)
+            (N): LayerNorm((10, 10, 10), eps=1e-05, elementwise_affine=True)
+          )
+        )
+
+    Args:
+        spatial_dims: number of spatial dimensions.
+        in_channels: number of input channels.
+        out_channels: number of output channels.
+        strides: convolution stride. Defaults to 1.
+        kernel_size: convolution kernel size. Defaults to 3.
+        adn_ordering: a string representing the ordering of activation, normalization, and dropout.
+            Defaults to "NDA".
+        act: activation type and arguments. Defaults to PReLU.
+        norm: feature normalization type and arguments. Defaults to instance norm.
+        dropout: dropout ratio. Defaults to no dropout.
+        dropout_dim: determine the spatial dimensions of dropout. Defaults to 1.
+
+            - When dropout_dim = 1, randomly zeroes some of the elements for each channel.
+            - When dropout_dim = 2, Randomly zeroes out entire channels (a channel is a 2D feature map).
+            - When dropout_dim = 3, Randomly zeroes out entire channels (a channel is a 3D feature map).
+
+            The value of dropout_dim should be no no larger than the value of `spatial_dims`.
+        dilation: dilation rate. Defaults to 1.
+        groups: controls the connections between inputs and outputs. Defaults to 1.
+        bias: whether to have a bias term. Defaults to True.
+        conv_only: whether to use the convolutional layer only. Defaults to False.
+        is_transposed: if True uses ConvTrans instead of Conv. Defaults to False.
+        padding: controls the amount of implicit zero-paddings on both sides for padding number of points
+            for each dimension. Defaults to None.
+        output_padding: controls the additional size added to one side of the output shape.
+            Defaults to None.
+
+    .. deprecated:: 0.6.0
+        ``dimensions`` is deprecated, use ``spatial_dims`` instead.
+
+    See also:
+
+        :py:class:`monai.networks.layers.Conv`
+        :py:class:`monai.networks.blocks.ADN`
+
+    """
+
+    @deprecated_arg(
+        name="dimensions", new_name="spatial_dims", since="0.6", msg_suffix="Please use `spatial_dims` instead."
+    )
+    def __init__(
+        self,
+        spatial_dims: int,
+        in_channels: int,
+        out_channels: int,
+        strides: Union[Sequence[int], int] = 1,
+        kernel_size: Union[Sequence[int], int] = 3,
+        adn_ordering: str = "NDA",
+        act: Optional[Union[Tuple, str]] = "PRELU",
+        norm: Optional[Union[Tuple, str]] = "INSTANCE",
+        dropout: Optional[Union[Tuple, str, float]] = None,
+        dropout_dim: Optional[int] = 1,
+        dilation: Union[Sequence[int], int] = 1,
+        groups: int = 1,
+        bias: bool = True,
+        conv_only: bool = False,
+        is_transposed: bool = False,
+        padding: Optional[Union[Sequence[int], int]] = None,
+        output_padding: Optional[Union[Sequence[int], int]] = None,
+        dimensions: Optional[int] = None,
+    ) -> None:
+        super().__init__()
+        self.dimensions = spatial_dims if dimensions is None else dimensions
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+        self.is_transposed = is_transposed
+        if padding is None:
+            padding = same_padding(kernel_size, dilation)
+        conv_type = Conv[Conv.CONVTRANS if is_transposed else Conv.CONV, self.dimensions]
+
+        conv: nn.Module
+        if is_transposed:
+            if output_padding is None:
+                output_padding = stride_minus_kernel_padding(1, strides)
+            conv = conv_type(
+                in_channels,
+                out_channels,
+                kernel_size=kernel_size,
+                stride=strides,
+                padding=padding,
+                output_padding=output_padding,
+                groups=groups,
+                bias=bias,
+                dilation=dilation,
+            )
+        else:
+            conv = conv_type(
+                in_channels,
+                out_channels,
+                kernel_size=kernel_size,
+                stride=strides,
+                padding=padding,
+                dilation=dilation,
+                groups=groups,
+                bias=bias,
+            )
+
+        self.add_module("conv", conv)
+
+        if not conv_only:
+            self.add_module(
+                "adn",
+                ADN(
+                    ordering=adn_ordering,
+                    in_channels=out_channels,
+                    act=act,
+                    norm=norm,
+                    norm_dim=self.dimensions,
+                    dropout=dropout,
+                    dropout_dim=dropout_dim,
+                ),
+            )
+
+
+class ResidualUnit(nn.Module):
+    """
+    Residual module with multiple convolutions and a residual connection.
+
+    For example:
+
+    .. code-block:: python
+
+        from monai.networks.blocks import ResidualUnit
+
+        convs = ResidualUnit(
+            spatial_dims=3,
+            in_channels=1,
+            out_channels=1,
+            adn_ordering="AN",
+            act=("prelu", {"init": 0.2}),
+            norm=("layer", {"normalized_shape": (10, 10, 10)}),
+        )
+        print(convs)
+
+    output::
+
+        ResidualUnit(
+          (conv): Sequential(
+            (unit0): Convolution(
+              (conv): Conv3d(1, 1, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+              (adn): ADN(
+                (A): PReLU(num_parameters=1)
+                (N): LayerNorm((10, 10, 10), eps=1e-05, elementwise_affine=True)
+              )
+            )
+            (unit1): Convolution(
+              (conv): Conv3d(1, 1, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+              (adn): ADN(
+                (A): PReLU(num_parameters=1)
+                (N): LayerNorm((10, 10, 10), eps=1e-05, elementwise_affine=True)
+              )
+            )
+          )
+          (residual): Identity()
+        )
+
+    Args:
+        spatial_dims: number of spatial dimensions.
+        in_channels: number of input channels.
+        out_channels: number of output channels.
+        strides: convolution stride. Defaults to 1.
+        kernel_size: convolution kernel size. Defaults to 3.
+        subunits: number of convolutions. Defaults to 2.
+        adn_ordering: a string representing the ordering of activation, normalization, and dropout.
+            Defaults to "NDA".
+        act: activation type and arguments. Defaults to PReLU.
+        norm: feature normalization type and arguments. Defaults to instance norm.
+        dropout: dropout ratio. Defaults to no dropout.
+        dropout_dim: determine the dimensions of dropout. Defaults to 1.
+
+            - When dropout_dim = 1, randomly zeroes some of the elements for each channel.
+            - When dropout_dim = 2, Randomly zero out entire channels (a channel is a 2D feature map).
+            - When dropout_dim = 3, Randomly zero out entire channels (a channel is a 3D feature map).
+
+            The value of dropout_dim should be no no larger than the value of `dimensions`.
+        dilation: dilation rate. Defaults to 1.
+        bias: whether to have a bias term. Defaults to True.
+        last_conv_only: for the last subunit, whether to use the convolutional layer only.
+            Defaults to False.
+        padding: controls the amount of implicit zero-paddings on both sides for padding number of points
+            for each dimension. Defaults to None.
+
+    .. deprecated:: 0.6.0
+        ``dimensions`` is deprecated, use ``spatial_dims`` instead.
+
+    See also:
+
+        :py:class:`monai.networks.blocks.Convolution`
+
+    """
+
+    @deprecated_arg(name="dimensions", since="0.6", msg_suffix="Please use `spatial_dims` instead.")
+    def __init__(
+        self,
+        spatial_dims: int,
+        in_channels: int,
+        out_channels: int,
+        strides: Union[Sequence[int], int] = 1,
+        kernel_size: Union[Sequence[int], int] = 3,
+        subunits: int = 2,
+        adn_ordering: str = "NDA",
+        act: Optional[Union[Tuple, str]] = "PRELU",
+        norm: Optional[Union[Tuple, str]] = "INSTANCE",
+        dropout: Optional[Union[Tuple, str, float]] = None,
+        dropout_dim: Optional[int] = 1,
+        dilation: Union[Sequence[int], int] = 1,
+        bias: bool = True,
+        last_conv_only: bool = False,
+        padding: Optional[Union[Sequence[int], int]] = None,
+        dimensions: Optional[int] = None,
+    ) -> None:
+        super().__init__()
+        self.dimensions = spatial_dims if dimensions is None else dimensions
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+        self.conv = nn.Sequential()
+        self.residual = nn.Identity()
+        if not padding:
+            padding = same_padding(kernel_size, dilation)
+        schannels = in_channels
+        sstrides = strides
+        subunits = max(1, subunits)
+
+        for su in range(subunits):
+            conv_only = last_conv_only and su == (subunits - 1)
+            unit = Convolution(
+                self.dimensions,
+                schannels,
+                out_channels,
+                strides=sstrides,
+                kernel_size=kernel_size,
+                adn_ordering=adn_ordering,
+                act=act,
+                norm=norm,
+                dropout=dropout,
+                dropout_dim=dropout_dim,
+                dilation=dilation,
+                bias=bias,
+                conv_only=conv_only,
+                padding=padding,
+            )
+
+            self.conv.add_module(f"unit{su:d}", unit)
+
+            # after first loop set channels and strides to what they should be for subsequent units
+            schannels = out_channels
+            sstrides = 1
+
+        # apply convolution to input to change number of output channels and size to match that coming from self.conv
+        if np.prod(strides) != 1 or in_channels != out_channels:
+            rkernel_size = kernel_size
+            rpadding = padding
+
+            if np.prod(strides) == 1:  # if only adapting number of channels a 1x1 kernel is used with no padding
+                rkernel_size = 1
+                rpadding = 0
+
+            conv_type = Conv[Conv.CONV, self.dimensions]
+            self.residual = conv_type(in_channels, out_channels, rkernel_size, strides, rpadding, bias=bias)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        res: torch.Tensor = self.residual(x)  # create the additive residual from x
+        cx: torch.Tensor = self.conv(x)  # apply x to sequence of operations
+        return cx + res  # add the residual to the output

my_container_sandbox/workspace/anaconda3/lib/python3.8/site-packages/monai/networks/blocks/crf.py

@@ -0,0 +1,120 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional
+
+import torch
+from torch.nn.functional import softmax
+
+from monai.networks.layers.filtering import PHLFilter
+from monai.networks.utils import meshgrid_ij
+
+__all__ = ["CRF"]
+
+
+class CRF(torch.nn.Module):
+    """
+    Conditional Random Field: Combines message passing with a class
+    compatibility convolution into an iterative process designed
+    to successively minimise the energy of the class labeling.
+
+    In this implementation, the message passing step is a weighted
+    combination of a gaussian filter and a bilateral filter.
+    The bilateral term is included to respect existing structure
+    within the reference tensor.
+
+    See:
+        https://arxiv.org/abs/1502.03240
+    """
+
+    def __init__(
+        self,
+        iterations: int = 5,
+        bilateral_weight: float = 1.0,
+        gaussian_weight: float = 1.0,
+        bilateral_spatial_sigma: float = 5.0,
+        bilateral_color_sigma: float = 0.5,
+        gaussian_spatial_sigma: float = 5.0,
+        update_factor: float = 3.0,
+        compatibility_matrix: Optional[torch.Tensor] = None,
+    ):
+        """
+        Args:
+            iterations: the number of iterations.
+            bilateral_weight: the weighting of the bilateral term in the message passing step.
+            gaussian_weight: the weighting of the gaussian term in the message passing step.
+            bilateral_spatial_sigma: standard deviation in spatial coordinates for the bilateral term.
+            bilateral_color_sigma: standard deviation in color space for the bilateral term.
+            gaussian_spatial_sigma: standard deviation in spatial coordinates for the gaussian term.
+            update_factor: determines the magnitude of each update.
+            compatibility_matrix: a matrix describing class compatibility,
+                should be NxN where N is the number of classes.
+        """
+        super().__init__()
+        self.iterations = iterations
+        self.bilateral_weight = bilateral_weight
+        self.gaussian_weight = gaussian_weight
+        self.bilateral_spatial_sigma = bilateral_spatial_sigma
+        self.bilateral_color_sigma = bilateral_color_sigma
+        self.gaussian_spatial_sigma = gaussian_spatial_sigma
+        self.update_factor = update_factor
+        self.compatibility_matrix = compatibility_matrix
+
+    def forward(self, input_tensor: torch.Tensor, reference_tensor: torch.Tensor):
+        """
+        Args:
+            input_tensor: tensor containing initial class logits.
+            reference_tensor: the reference tensor used to guide the message passing.
+
+        Returns:
+            output (torch.Tensor): output tensor.
+        """
+
+        # constructing spatial feature tensor
+        spatial_features = _create_coordinate_tensor(reference_tensor)
+
+        # constructing final feature tensors for bilateral and gaussian kernel
+        bilateral_features = torch.cat(
+            [spatial_features / self.bilateral_spatial_sigma, reference_tensor / self.bilateral_color_sigma], dim=1
+        )
+        gaussian_features = spatial_features / self.gaussian_spatial_sigma
+
+        # setting up output tensor
+        output_tensor = softmax(input_tensor, dim=1)
+
+        # mean field loop
+        for _ in range(self.iterations):
+
+            # message passing step for both kernels
+            bilateral_output = PHLFilter.apply(output_tensor, bilateral_features)
+            gaussian_output = PHLFilter.apply(output_tensor, gaussian_features)
+
+            # combining filter outputs
+            combined_output = self.bilateral_weight * bilateral_output + self.gaussian_weight * gaussian_output
+
+            # optionally running a compatibility transform
+            if self.compatibility_matrix is not None:
+                flat = combined_output.flatten(start_dim=2).permute(0, 2, 1)
+                flat = torch.matmul(flat, self.compatibility_matrix)
+                combined_output = flat.permute(0, 2, 1).reshape(combined_output.shape)
+
+            # update and normalize
+            output_tensor = softmax(input_tensor + self.update_factor * combined_output, dim=1)
+
+        return output_tensor
+
+
+# helper methods
+def _create_coordinate_tensor(tensor):
+    axes = [torch.arange(tensor.size(i)) for i in range(2, tensor.dim())]
+    grids = meshgrid_ij(axes)
+    coords = torch.stack(grids).to(device=tensor.device, dtype=tensor.dtype)
+    return torch.stack(tensor.size(0) * [coords], dim=0)

my_container_sandbox/workspace/anaconda3/lib/python3.8/site-packages/monai/networks/blocks/dints_block.py

@@ -0,0 +1,272 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from typing import Tuple, Union
+
+import torch
+
+from monai.networks.layers.factories import Conv
+from monai.networks.layers.utils import get_act_layer, get_norm_layer
+
+__all__ = ["FactorizedIncreaseBlock", "FactorizedReduceBlock", "P3DActiConvNormBlock", "ActiConvNormBlock"]
+
+
+class FactorizedIncreaseBlock(torch.nn.Sequential):
+    """
+    Up-sampling the features by two using linear interpolation and convolutions.
+    """
+
+    def __init__(
+        self,
+        in_channel: int,
+        out_channel: int,
+        spatial_dims: int = 3,
+        act_name: Union[Tuple, str] = "RELU",
+        norm_name: Union[Tuple, str] = ("INSTANCE", {"affine": True}),
+    ):
+        """
+        Args:
+            in_channel: number of input channels
+            out_channel: number of output channels
+            spatial_dims: number of spatial dimensions
+            act_name: activation layer type and arguments.
+            norm_name: feature normalization type and arguments.
+        """
+        super().__init__()
+        self._in_channel = in_channel
+        self._out_channel = out_channel
+        self._spatial_dims = spatial_dims
+        if self._spatial_dims not in (2, 3):
+            raise ValueError("spatial_dims must be 2 or 3.")
+
+        conv_type = Conv[Conv.CONV, self._spatial_dims]
+        mode = "trilinear" if self._spatial_dims == 3 else "bilinear"
+        self.add_module("up", torch.nn.Upsample(scale_factor=2, mode=mode, align_corners=True))
+        self.add_module("acti", get_act_layer(name=act_name))
+        self.add_module(
+            "conv",
+            conv_type(
+                in_channels=self._in_channel,
+                out_channels=self._out_channel,
+                kernel_size=1,
+                stride=1,
+                padding=0,
+                groups=1,
+                bias=False,
+                dilation=1,
+            ),
+        )
+        self.add_module(
+            "norm", get_norm_layer(name=norm_name, spatial_dims=self._spatial_dims, channels=self._out_channel)
+        )
+
+
+class FactorizedReduceBlock(torch.nn.Module):
+    """
+    Down-sampling the feature by 2 using stride.
+    The length along each spatial dimension must be a multiple of 2.
+    """
+
+    def __init__(
+        self,
+        in_channel: int,
+        out_channel: int,
+        spatial_dims: int = 3,
+        act_name: Union[Tuple, str] = "RELU",
+        norm_name: Union[Tuple, str] = ("INSTANCE", {"affine": True}),
+    ):
+        """
+        Args:
+            in_channel: number of input channels
+            out_channel: number of output channels.
+            spatial_dims: number of spatial dimensions.
+            act_name: activation layer type and arguments.
+            norm_name: feature normalization type and arguments.
+        """
+        super().__init__()
+        self._in_channel = in_channel
+        self._out_channel = out_channel
+        self._spatial_dims = spatial_dims
+        if self._spatial_dims not in (2, 3):
+            raise ValueError("spatial_dims must be 2 or 3.")
+
+        conv_type = Conv[Conv.CONV, self._spatial_dims]
+
+        self.act = get_act_layer(name=act_name)
+        self.conv_1 = conv_type(
+            in_channels=self._in_channel,
+            out_channels=self._out_channel // 2,
+            kernel_size=1,
+            stride=2,
+            padding=0,
+            groups=1,
+            bias=False,
+            dilation=1,
+        )
+        self.conv_2 = conv_type(
+            in_channels=self._in_channel,
+            out_channels=self._out_channel - self._out_channel // 2,
+            kernel_size=1,
+            stride=2,
+            padding=0,
+            groups=1,
+            bias=False,
+            dilation=1,
+        )
+        self.norm = get_norm_layer(name=norm_name, spatial_dims=self._spatial_dims, channels=self._out_channel)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        """
+        The length along each spatial dimension must be a multiple of 2.
+        """
+        x = self.act(x)
+        if self._spatial_dims == 3:
+            out = torch.cat([self.conv_1(x), self.conv_2(x[:, :, 1:, 1:, 1:])], dim=1)
+        else:
+            out = torch.cat([self.conv_1(x), self.conv_2(x[:, :, 1:, 1:])], dim=1)
+        out = self.norm(out)
+        return out
+
+
+class P3DActiConvNormBlock(torch.nn.Sequential):
+    """
+    -- (act) -- (conv) -- (norm) --
+    """
+
+    def __init__(
+        self,
+        in_channel: int,
+        out_channel: int,
+        kernel_size: int,
+        padding: int,
+        mode: int = 0,
+        act_name: Union[Tuple, str] = "RELU",
+        norm_name: Union[Tuple, str] = ("INSTANCE", {"affine": True}),
+    ):
+        """
+        Args:
+            in_channel: number of input channels.
+            out_channel: number of output channels.
+            kernel_size: kernel size to be expanded to 3D.
+            padding: padding size to be expanded to 3D.
+            mode: mode for the anisotropic kernels:
+
+                - 0: ``(k, k, 1)``, ``(1, 1, k)``,
+                - 1: ``(k, 1, k)``, ``(1, k, 1)``,
+                - 2: ``(1, k, k)``. ``(k, 1, 1)``.
+
+            act_name: activation layer type and arguments.
+            norm_name: feature normalization type and arguments.
+        """
+        super().__init__()
+        self._in_channel = in_channel
+        self._out_channel = out_channel
+        self._p3dmode = int(mode)
+
+        conv_type = Conv[Conv.CONV, 3]
+
+        if self._p3dmode == 0:  # (k, k, 1), (1, 1, k)
+            kernel_size0 = (kernel_size, kernel_size, 1)
+            kernel_size1 = (1, 1, kernel_size)
+            padding0 = (padding, padding, 0)
+            padding1 = (0, 0, padding)
+        elif self._p3dmode == 1:  # (k, 1, k), (1, k, 1)
+            kernel_size0 = (kernel_size, 1, kernel_size)
+            kernel_size1 = (1, kernel_size, 1)
+            padding0 = (padding, 0, padding)
+            padding1 = (0, padding, 0)
+        elif self._p3dmode == 2:  # (1, k, k), (k, 1, 1)
+            kernel_size0 = (1, kernel_size, kernel_size)
+            kernel_size1 = (kernel_size, 1, 1)
+            padding0 = (0, padding, padding)
+            padding1 = (padding, 0, 0)
+        else:
+            raise ValueError("`mode` must be 0, 1, or 2.")
+
+        self.add_module("acti", get_act_layer(name=act_name))
+        self.add_module(
+            "conv",
+            conv_type(
+                in_channels=self._in_channel,
+                out_channels=self._in_channel,
+                kernel_size=kernel_size0,
+                stride=1,
+                padding=padding0,
+                groups=1,
+                bias=False,
+                dilation=1,
+            ),
+        )
+        self.add_module(
+            "conv_1",
+            conv_type(
+                in_channels=self._in_channel,
+                out_channels=self._out_channel,
+                kernel_size=kernel_size1,
+                stride=1,
+                padding=padding1,
+                groups=1,
+                bias=False,
+                dilation=1,
+            ),
+        )
+        self.add_module("norm", get_norm_layer(name=norm_name, spatial_dims=3, channels=self._out_channel))
+
+
+class ActiConvNormBlock(torch.nn.Sequential):
+    """
+    -- (Acti) -- (Conv) -- (Norm) --
+    """
+
+    def __init__(
+        self,
+        in_channel: int,
+        out_channel: int,
+        kernel_size: int = 3,
+        padding: int = 1,
+        spatial_dims: int = 3,
+        act_name: Union[Tuple, str] = "RELU",
+        norm_name: Union[Tuple, str] = ("INSTANCE", {"affine": True}),
+    ):
+        """
+        Args:
+            in_channel: number of input channels.
+            out_channel: number of output channels.
+            kernel_size: kernel size of the convolution.
+            padding: padding size of the convolution.
+            spatial_dims: number of spatial dimensions.
+            act_name: activation layer type and arguments.
+            norm_name: feature normalization type and arguments.
+        """
+        super().__init__()
+        self._in_channel = in_channel
+        self._out_channel = out_channel
+        self._spatial_dims = spatial_dims
+
+        conv_type = Conv[Conv.CONV, self._spatial_dims]
+        self.add_module("acti", get_act_layer(name=act_name))
+        self.add_module(
+            "conv",
+            conv_type(
+                in_channels=self._in_channel,
+                out_channels=self._out_channel,
+                kernel_size=kernel_size,
+                stride=1,
+                padding=padding,
+                groups=1,
+                bias=False,
+                dilation=1,
+            ),
+        )
+        self.add_module(
+            "norm", get_norm_layer(name=norm_name, spatial_dims=self._spatial_dims, channels=self._out_channel)
+        )

my_container_sandbox/workspace/anaconda3/lib/python3.8/site-packages/monai/networks/blocks/downsample.py

@@ -0,0 +1,61 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional, Sequence, Union
+
+import torch
+import torch.nn as nn
+
+from monai.networks.layers.factories import Pool
+from monai.utils import ensure_tuple_rep
+
+
+class MaxAvgPool(nn.Module):
+    """
+    Downsample with both maxpooling and avgpooling,
+    double the channel size by concatenating the downsampled feature maps.
+    """
+
+    def __init__(
+        self,
+        spatial_dims: int,
+        kernel_size: Union[Sequence[int], int],
+        stride: Optional[Union[Sequence[int], int]] = None,
+        padding: Union[Sequence[int], int] = 0,
+        ceil_mode: bool = False,
+    ) -> None:
+        """
+        Args:
+            spatial_dims: number of spatial dimensions of the input image.
+            kernel_size: the kernel size of both pooling operations.
+            stride: the stride of the window. Default value is `kernel_size`.
+            padding: implicit zero padding to be added to both pooling operations.
+            ceil_mode: when True, will use ceil instead of floor to compute the output shape.
+        """
+        super().__init__()
+        _params = {
+            "kernel_size": ensure_tuple_rep(kernel_size, spatial_dims),
+            "stride": None if stride is None else ensure_tuple_rep(stride, spatial_dims),
+            "padding": ensure_tuple_rep(padding, spatial_dims),
+            "ceil_mode": ceil_mode,
+        }
+        self.max_pool = Pool[Pool.MAX, spatial_dims](**_params)
+        self.avg_pool = Pool[Pool.AVG, spatial_dims](**_params)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        """
+        Args:
+            x: Tensor in shape (batch, channel, spatial_1[, spatial_2, ...]).
+
+        Returns:
+            Tensor in shape (batch, 2*channel, spatial_1[, spatial_2, ...]).
+        """
+        return torch.cat([self.max_pool(x), self.avg_pool(x)], dim=1)

my_container_sandbox/workspace/anaconda3/lib/python3.8/site-packages/monai/networks/blocks/dynunet_block.py

@@ -0,0 +1,288 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional, Sequence, Tuple, Union
+
+import numpy as np
+import torch
+import torch.nn as nn
+
+from monai.networks.blocks.convolutions import Convolution
+from monai.networks.layers.factories import Act, Norm
+from monai.networks.layers.utils import get_act_layer, get_norm_layer
+
+
+class UnetResBlock(nn.Module):
+    """
+    A skip-connection based module that can be used for DynUNet, based on:
+    `Automated Design of Deep Learning Methods for Biomedical Image Segmentation <https://arxiv.org/abs/1904.08128>`_.
+    `nnU-Net: Self-adapting Framework for U-Net-Based Medical Image Segmentation <https://arxiv.org/abs/1809.10486>`_.
+
+    Args:
+        spatial_dims: number of spatial dimensions.
+        in_channels: number of input channels.
+        out_channels: number of output channels.
+        kernel_size: convolution kernel size.
+        stride: convolution stride.
+        norm_name: feature normalization type and arguments.
+        act_name: activation layer type and arguments.
+        dropout: dropout probability.
+
+    """
+
+    def __init__(
+        self,
+        spatial_dims: int,
+        in_channels: int,
+        out_channels: int,
+        kernel_size: Union[Sequence[int], int],
+        stride: Union[Sequence[int], int],
+        norm_name: Union[Tuple, str],
+        act_name: Union[Tuple, str] = ("leakyrelu", {"inplace": True, "negative_slope": 0.01}),
+        dropout: Optional[Union[Tuple, str, float]] = None,
+    ):
+        super().__init__()
+        self.conv1 = get_conv_layer(
+            spatial_dims,
+            in_channels,
+            out_channels,
+            kernel_size=kernel_size,
+            stride=stride,
+            dropout=dropout,
+            conv_only=True,
+        )
+        self.conv2 = get_conv_layer(
+            spatial_dims, out_channels, out_channels, kernel_size=kernel_size, stride=1, dropout=dropout, conv_only=True
+        )
+        self.lrelu = get_act_layer(name=act_name)
+        self.norm1 = get_norm_layer(name=norm_name, spatial_dims=spatial_dims, channels=out_channels)
+        self.norm2 = get_norm_layer(name=norm_name, spatial_dims=spatial_dims, channels=out_channels)
+        self.downsample = in_channels != out_channels
+        stride_np = np.atleast_1d(stride)
+        if not np.all(stride_np == 1):
+            self.downsample = True
+        if self.downsample:
+            self.conv3 = get_conv_layer(
+                spatial_dims, in_channels, out_channels, kernel_size=1, stride=stride, dropout=dropout, conv_only=True
+            )
+            self.norm3 = get_norm_layer(name=norm_name, spatial_dims=spatial_dims, channels=out_channels)
+
+    def forward(self, inp):
+        residual = inp
+        out = self.conv1(inp)
+        out = self.norm1(out)
+        out = self.lrelu(out)
+        out = self.conv2(out)
+        out = self.norm2(out)
+        if hasattr(self, "conv3"):
+            residual = self.conv3(residual)
+        if hasattr(self, "norm3"):
+            residual = self.norm3(residual)
+        out += residual
+        out = self.lrelu(out)
+        return out
+
+
+class UnetBasicBlock(nn.Module):
+    """
+    A CNN module module that can be used for DynUNet, based on:
+    `Automated Design of Deep Learning Methods for Biomedical Image Segmentation <https://arxiv.org/abs/1904.08128>`_.
+    `nnU-Net: Self-adapting Framework for U-Net-Based Medical Image Segmentation <https://arxiv.org/abs/1809.10486>`_.
+
+    Args:
+        spatial_dims: number of spatial dimensions.
+        in_channels: number of input channels.
+        out_channels: number of output channels.
+        kernel_size: convolution kernel size.
+        stride: convolution stride.
+        norm_name: feature normalization type and arguments.
+        act_name: activation layer type and arguments.
+        dropout: dropout probability.
+
+    """
+
+    def __init__(
+        self,
+        spatial_dims: int,
+        in_channels: int,
+        out_channels: int,
+        kernel_size: Union[Sequence[int], int],
+        stride: Union[Sequence[int], int],
+        norm_name: Union[Tuple, str],
+        act_name: Union[Tuple, str] = ("leakyrelu", {"inplace": True, "negative_slope": 0.01}),
+        dropout: Optional[Union[Tuple, str, float]] = None,
+    ):
+        super().__init__()
+        self.conv1 = get_conv_layer(
+            spatial_dims,
+            in_channels,
+            out_channels,
+            kernel_size=kernel_size,
+            stride=stride,
+            dropout=dropout,
+            conv_only=True,
+        )
+        self.conv2 = get_conv_layer(
+            spatial_dims, out_channels, out_channels, kernel_size=kernel_size, stride=1, dropout=dropout, conv_only=True
+        )
+        self.lrelu = get_act_layer(name=act_name)
+        self.norm1 = get_norm_layer(name=norm_name, spatial_dims=spatial_dims, channels=out_channels)
+        self.norm2 = get_norm_layer(name=norm_name, spatial_dims=spatial_dims, channels=out_channels)
+
+    def forward(self, inp):
+        out = self.conv1(inp)
+        out = self.norm1(out)
+        out = self.lrelu(out)
+        out = self.conv2(out)
+        out = self.norm2(out)
+        out = self.lrelu(out)
+        return out
+
+
+class UnetUpBlock(nn.Module):
+    """
+    An upsampling module that can be used for DynUNet, based on:
+    `Automated Design of Deep Learning Methods for Biomedical Image Segmentation <https://arxiv.org/abs/1904.08128>`_.
+    `nnU-Net: Self-adapting Framework for U-Net-Based Medical Image Segmentation <https://arxiv.org/abs/1809.10486>`_.
+
+    Args:
+        spatial_dims: number of spatial dimensions.
+        in_channels: number of input channels.
+        out_channels: number of output channels.
+        kernel_size: convolution kernel size.
+        stride: convolution stride.
+        upsample_kernel_size: convolution kernel size for transposed convolution layers.
+        norm_name: feature normalization type and arguments.
+        act_name: activation layer type and arguments.
+        dropout: dropout probability.
+        trans_bias: transposed convolution bias.
+
+    """
+
+    def __init__(
+        self,
+        spatial_dims: int,
+        in_channels: int,
+        out_channels: int,
+        kernel_size: Union[Sequence[int], int],
+        stride: Union[Sequence[int], int],
+        upsample_kernel_size: Union[Sequence[int], int],
+        norm_name: Union[Tuple, str],
+        act_name: Union[Tuple, str] = ("leakyrelu", {"inplace": True, "negative_slope": 0.01}),
+        dropout: Optional[Union[Tuple, str, float]] = None,
+        trans_bias: bool = False,
+    ):
+        super().__init__()
+        upsample_stride = upsample_kernel_size
+        self.transp_conv = get_conv_layer(
+            spatial_dims,
+            in_channels,
+            out_channels,
+            kernel_size=upsample_kernel_size,
+            stride=upsample_stride,
+            dropout=dropout,
+            bias=trans_bias,
+            conv_only=True,
+            is_transposed=True,
+        )
+        self.conv_block = UnetBasicBlock(
+            spatial_dims,
+            out_channels + out_channels,
+            out_channels,
+            kernel_size=kernel_size,
+            stride=1,
+            dropout=dropout,
+            norm_name=norm_name,
+            act_name=act_name,
+        )
+
+    def forward(self, inp, skip):
+        # number of channels for skip should equals to out_channels
+        out = self.transp_conv(inp)
+        out = torch.cat((out, skip), dim=1)
+        out = self.conv_block(out)
+        return out
+
+
+class UnetOutBlock(nn.Module):
+    def __init__(
+        self, spatial_dims: int, in_channels: int, out_channels: int, dropout: Optional[Union[Tuple, str, float]] = None
+    ):
+        super().__init__()
+        self.conv = get_conv_layer(
+            spatial_dims, in_channels, out_channels, kernel_size=1, stride=1, dropout=dropout, bias=True, conv_only=True
+        )
+
+    def forward(self, inp):
+        return self.conv(inp)
+
+
+def get_conv_layer(
+    spatial_dims: int,
+    in_channels: int,
+    out_channels: int,
+    kernel_size: Union[Sequence[int], int] = 3,
+    stride: Union[Sequence[int], int] = 1,
+    act: Optional[Union[Tuple, str]] = Act.PRELU,
+    norm: Union[Tuple, str] = Norm.INSTANCE,
+    dropout: Optional[Union[Tuple, str, float]] = None,
+    bias: bool = False,
+    conv_only: bool = True,
+    is_transposed: bool = False,
+):
+    padding = get_padding(kernel_size, stride)
+    output_padding = None
+    if is_transposed:
+        output_padding = get_output_padding(kernel_size, stride, padding)
+    return Convolution(
+        spatial_dims,
+        in_channels,
+        out_channels,
+        strides=stride,
+        kernel_size=kernel_size,
+        act=act,
+        norm=norm,
+        dropout=dropout,
+        bias=bias,
+        conv_only=conv_only,
+        is_transposed=is_transposed,
+        padding=padding,
+        output_padding=output_padding,
+    )
+
+
+def get_padding(
+    kernel_size: Union[Sequence[int], int], stride: Union[Sequence[int], int]
+) -> Union[Tuple[int, ...], int]:
+
+    kernel_size_np = np.atleast_1d(kernel_size)
+    stride_np = np.atleast_1d(stride)
+    padding_np = (kernel_size_np - stride_np + 1) / 2
+    if np.min(padding_np) < 0:
+        raise AssertionError("padding value should not be negative, please change the kernel size and/or stride.")
+    padding = tuple(int(p) for p in padding_np)
+
+    return padding if len(padding) > 1 else padding[0]
+
+
+def get_output_padding(
+    kernel_size: Union[Sequence[int], int], stride: Union[Sequence[int], int], padding: Union[Sequence[int], int]
+) -> Union[Tuple[int, ...], int]:
+    kernel_size_np = np.atleast_1d(kernel_size)
+    stride_np = np.atleast_1d(stride)
+    padding_np = np.atleast_1d(padding)
+
+    out_padding_np = 2 * padding_np + stride_np - kernel_size_np
+    if np.min(out_padding_np) < 0:
+        raise AssertionError("out_padding value should not be negative, please change the kernel size and/or stride.")
+    out_padding = tuple(int(p) for p in out_padding_np)
+
+    return out_padding if len(out_padding) > 1 else out_padding[0]

my_container_sandbox/workspace/anaconda3/lib/python3.8/site-packages/monai/networks/blocks/fcn.py

@@ -0,0 +1,242 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Type
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from monai.networks.blocks.convolutions import Convolution
+from monai.networks.blocks.upsample import UpSample
+from monai.networks.layers.factories import Act, Conv, Norm
+from monai.utils import optional_import
+
+models, _ = optional_import("torchvision", name="models")
+
+
+class GCN(nn.Module):
+    """
+    The Global Convolutional Network module using large 1D
+    Kx1 and 1xK kernels to represent 2D kernels.
+    """
+
+    def __init__(self, inplanes: int, planes: int, ks: int = 7):
+        """
+        Args:
+            inplanes: number of input channels.
+            planes: number of output channels.
+            ks: kernel size for one dimension. Defaults to 7.
+        """
+        super().__init__()
+
+        conv2d_type: Type[nn.Conv2d] = Conv[Conv.CONV, 2]
+        self.conv_l1 = conv2d_type(in_channels=inplanes, out_channels=planes, kernel_size=(ks, 1), padding=(ks // 2, 0))
+        self.conv_l2 = conv2d_type(in_channels=planes, out_channels=planes, kernel_size=(1, ks), padding=(0, ks // 2))
+        self.conv_r1 = conv2d_type(in_channels=inplanes, out_channels=planes, kernel_size=(1, ks), padding=(0, ks // 2))
+        self.conv_r2 = conv2d_type(in_channels=planes, out_channels=planes, kernel_size=(ks, 1), padding=(ks // 2, 0))
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        """
+        Args:
+            x: in shape (batch, inplanes, spatial_1, spatial_2).
+        """
+        x_l = self.conv_l1(x)
+        x_l = self.conv_l2(x_l)
+        x_r = self.conv_r1(x)
+        x_r = self.conv_r2(x_r)
+        x = x_l + x_r
+        return x
+
+
+class Refine(nn.Module):
+    """
+    Simple residual block to refine the details of the activation maps.
+    """
+
+    def __init__(self, planes: int):
+        """
+        Args:
+            planes: number of input channels.
+        """
+        super().__init__()
+
+        relu_type: Type[nn.ReLU] = Act[Act.RELU]
+        conv2d_type: Type[nn.Conv2d] = Conv[Conv.CONV, 2]
+        norm2d_type: Type[nn.BatchNorm2d] = Norm[Norm.BATCH, 2]
+
+        self.bn = norm2d_type(num_features=planes)
+        self.relu = relu_type(inplace=True)
+        self.conv1 = conv2d_type(in_channels=planes, out_channels=planes, kernel_size=3, padding=1)
+        self.conv2 = conv2d_type(in_channels=planes, out_channels=planes, kernel_size=3, padding=1)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        """
+        Args:
+            x: in shape (batch, planes, spatial_1, spatial_2).
+        """
+        residual = x
+        x = self.bn(x)
+        x = self.relu(x)
+        x = self.conv1(x)
+        x = self.bn(x)
+        x = self.relu(x)
+        x = self.conv2(x)
+
+        return residual + x
+
+
+class FCN(nn.Module):
+    """
+    2D FCN network with 3 input channels. The small decoder is built
+    with the GCN and Refine modules.
+    The code is adapted from `lsqshr's official 2D code <https://github.com/lsqshr/AH-Net/blob/master/net2d.py>`_.
+
+    Args:
+        out_channels: number of output channels. Defaults to 1.
+        upsample_mode: [``"transpose"``, ``"bilinear"``]
+            The mode of upsampling manipulations.
+            Using the second mode cannot guarantee the model's reproducibility. Defaults to ``bilinear``.
+
+            - ``transpose``, uses transposed convolution layers.
+            - ``bilinear``, uses bilinear interpolation.
+
+        pretrained: If True, returns a model pre-trained on ImageNet
+        progress: If True, displays a progress bar of the download to stderr.
+    """
+
+    def __init__(
+        self, out_channels: int = 1, upsample_mode: str = "bilinear", pretrained: bool = True, progress: bool = True
+    ):
+        super().__init__()
+
+        conv2d_type: Type[nn.Conv2d] = Conv[Conv.CONV, 2]
+
+        self.upsample_mode = upsample_mode
+        self.conv2d_type = conv2d_type
+        self.out_channels = out_channels
+        resnet = models.resnet50(pretrained=pretrained, progress=progress)
+
+        self.conv1 = resnet.conv1
+        self.bn0 = resnet.bn1
+        self.relu = resnet.relu
+        self.maxpool = resnet.maxpool
+
+        self.layer1 = resnet.layer1
+        self.layer2 = resnet.layer2
+        self.layer3 = resnet.layer3
+        self.layer4 = resnet.layer4
+
+        self.gcn1 = GCN(2048, self.out_channels)
+        self.gcn2 = GCN(1024, self.out_channels)
+        self.gcn3 = GCN(512, self.out_channels)
+        self.gcn4 = GCN(64, self.out_channels)
+        self.gcn5 = GCN(64, self.out_channels)
+
+        self.refine1 = Refine(self.out_channels)
+        self.refine2 = Refine(self.out_channels)
+        self.refine3 = Refine(self.out_channels)
+        self.refine4 = Refine(self.out_channels)
+        self.refine5 = Refine(self.out_channels)
+        self.refine6 = Refine(self.out_channels)
+        self.refine7 = Refine(self.out_channels)
+        self.refine8 = Refine(self.out_channels)
+        self.refine9 = Refine(self.out_channels)
+        self.refine10 = Refine(self.out_channels)
+        self.transformer = self.conv2d_type(in_channels=256, out_channels=64, kernel_size=1)
+
+        if self.upsample_mode == "transpose":
+            self.up_conv = UpSample(spatial_dims=2, in_channels=self.out_channels, scale_factor=2, mode="deconv")
+
+    def forward(self, x: torch.Tensor):
+        """
+        Args:
+            x: in shape (batch, 3, spatial_1, spatial_2).
+        """
+        org_input = x
+        x = self.conv1(x)
+        x = self.bn0(x)
+        x = self.relu(x)
+        conv_x = x
+        x = self.maxpool(x)
+        pool_x = x
+
+        fm1 = self.layer1(x)
+        fm2 = self.layer2(fm1)
+        fm3 = self.layer3(fm2)
+        fm4 = self.layer4(fm3)
+
+        gcfm1 = self.refine1(self.gcn1(fm4))
+        gcfm2 = self.refine2(self.gcn2(fm3))
+        gcfm3 = self.refine3(self.gcn3(fm2))
+        gcfm4 = self.refine4(self.gcn4(pool_x))
+        gcfm5 = self.refine5(self.gcn5(conv_x))
+
+        if self.upsample_mode == "transpose":
+            fs1 = self.refine6(self.up_conv(gcfm1) + gcfm2)
+            fs2 = self.refine7(self.up_conv(fs1) + gcfm3)
+            fs3 = self.refine8(self.up_conv(fs2) + gcfm4)
+            fs4 = self.refine9(self.up_conv(fs3) + gcfm5)
+            return self.refine10(self.up_conv(fs4))
+        fs1 = self.refine6(F.interpolate(gcfm1, fm3.size()[2:], mode=self.upsample_mode, align_corners=True) + gcfm2)
+        fs2 = self.refine7(F.interpolate(fs1, fm2.size()[2:], mode=self.upsample_mode, align_corners=True) + gcfm3)
+        fs3 = self.refine8(F.interpolate(fs2, pool_x.size()[2:], mode=self.upsample_mode, align_corners=True) + gcfm4)
+        fs4 = self.refine9(F.interpolate(fs3, conv_x.size()[2:], mode=self.upsample_mode, align_corners=True) + gcfm5)
+        return self.refine10(F.interpolate(fs4, org_input.size()[2:], mode=self.upsample_mode, align_corners=True))
+
+
+class MCFCN(FCN):
+    """
+    The multi-channel version of the 2D FCN module.
+    Adds a projection layer to take arbitrary number of inputs.
+
+    Args:
+        in_channels: number of input channels. Defaults to 3.
+        out_channels: number of output channels. Defaults to 1.
+        upsample_mode: [``"transpose"``, ``"bilinear"``]
+            The mode of upsampling manipulations.
+            Using the second mode cannot guarantee the model's reproducibility. Defaults to ``bilinear``.
+
+            - ``transpose``, uses transposed convolution layers.
+            - ``bilinear``, uses bilinear interpolate.
+        pretrained: If True, returns a model pre-trained on ImageNet
+        progress: If True, displays a progress bar of the download to stderr.
+    """
+
+    def __init__(
+        self,
+        in_channels: int = 3,
+        out_channels: int = 1,
+        upsample_mode: str = "bilinear",
+        pretrained: bool = True,
+        progress: bool = True,
+    ):
+        super().__init__(
+            out_channels=out_channels, upsample_mode=upsample_mode, pretrained=pretrained, progress=progress
+        )
+
+        self.init_proj = Convolution(
+            spatial_dims=2,
+            in_channels=in_channels,
+            out_channels=3,
+            kernel_size=1,
+            act=("relu", {"inplace": True}),
+            norm=Norm.BATCH,
+            bias=False,
+        )
+
+    def forward(self, x: torch.Tensor):
+        """
+        Args:
+            x: in shape (batch, in_channels, spatial_1, spatial_2).
+        """
+        x = self.init_proj(x)
+        return super().forward(x)

my_container_sandbox/workspace/anaconda3/lib/python3.8/site-packages/monai/networks/blocks/feature_pyramid_network.py

@@ -0,0 +1,263 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# =========================================================================
+# Adapted from https://github.com/pytorch/vision/blob/release/0.12/torchvision/ops/feature_pyramid_network.py
+# which has the following license...
+# https://github.com/pytorch/vision/blob/main/LICENSE
+#
+# BSD 3-Clause License
+
+# Copyright (c) Soumith Chintala 2016,
+# All rights reserved.
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""
+This script is modified from from torchvision to support N-D images,
+by overriding the definition of convolutional layers and pooling layers.
+
+https://github.com/pytorch/vision/blob/release/0.12/torchvision/ops/feature_pyramid_network.py
+"""
+
+from collections import OrderedDict
+from typing import Callable, Dict, List, Optional, Tuple, Type, Union
+
+import torch.nn.functional as F
+from torch import Tensor, nn
+
+from monai.networks.layers.factories import Conv, Pool
+
+__all__ = ["ExtraFPNBlock", "LastLevelMaxPool", "LastLevelP6P7", "FeaturePyramidNetwork"]
+
+
+class ExtraFPNBlock(nn.Module):
+    """
+    Base class for the extra block in the FPN.
+
+    Same code as https://github.com/pytorch/vision/blob/release/0.12/torchvision/ops/feature_pyramid_network.py
+    """
+
+    def forward(self, results: List[Tensor], x: List[Tensor], names: List[str]) -> Tuple[List[Tensor], List[str]]:
+        """
+        Compute extended set of results of the FPN and their names.
+
+        Args:
+            results: the result of the FPN
+            x: the original feature maps
+            names: the names for each one of the original feature maps
+
+        Returns:
+            - the extended set of results of the FPN
+            - the extended set of names for the results
+        """
+        pass
+
+
+class LastLevelMaxPool(ExtraFPNBlock):
+    """
+    Applies a max_pool2d or max_pool3d on top of the last feature map. Serves as an ``extra_blocks``
+    in :class:`~monai.networks.blocks.feature_pyramid_network.FeaturePyramidNetwork` .
+    """
+
+    def __init__(self, spatial_dims: int):
+        super().__init__()
+        pool_type: Type[Union[nn.MaxPool1d, nn.MaxPool2d, nn.MaxPool3d]] = Pool[Pool.MAX, spatial_dims]
+        self.maxpool = pool_type(kernel_size=1, stride=2, padding=0)
+
+    def forward(self, results: List[Tensor], x: List[Tensor], names: List[str]) -> Tuple[List[Tensor], List[str]]:
+        names.append("pool")
+        results.append(self.maxpool(results[-1]))
+        return results, names
+
+
+class LastLevelP6P7(ExtraFPNBlock):
+    """
+    This module is used in RetinaNet to generate extra layers, P6 and P7.
+    Serves as an ``extra_blocks``
+    in :class:`~monai.networks.blocks.feature_pyramid_network.FeaturePyramidNetwork` .
+    """
+
+    def __init__(self, spatial_dims: int, in_channels: int, out_channels: int):
+        super().__init__()
+        conv_type: Callable = Conv[Conv.CONV, spatial_dims]
+        self.p6 = conv_type(in_channels, out_channels, kernel_size=3, stride=2, padding=1)
+        self.p7 = conv_type(out_channels, out_channels, kernel_size=3, stride=2, padding=1)
+        for module in [self.p6, self.p7]:
+            nn.init.kaiming_uniform_(module.weight, a=1)
+            nn.init.constant_(module.bias, 0)
+        self.use_P5 = in_channels == out_channels
+
+    def forward(self, results: List[Tensor], x: List[Tensor], names: List[str]) -> Tuple[List[Tensor], List[str]]:
+        p5, c5 = results[-1], x[-1]
+        x5 = p5 if self.use_P5 else c5
+        p6 = self.p6(x5)
+        p7 = self.p7(F.relu(p6))
+        results.extend([p6, p7])
+        names.extend(["p6", "p7"])
+        return results, names
+
+
+class FeaturePyramidNetwork(nn.Module):
+    """
+    Module that adds a FPN from on top of a set of feature maps. This is based on
+    `"Feature Pyramid Network for Object Detection" <https://arxiv.org/abs/1612.03144>`_.
+
+    The feature maps are currently supposed to be in increasing depth
+    order.
+
+    The input to the model is expected to be an OrderedDict[Tensor], containing
+    the feature maps on top of which the FPN will be added.
+
+    Args:
+        spatial_dims: 2D or 3D images
+        in_channels_list: number of channels for each feature map that
+            is passed to the module
+        out_channels: number of channels of the FPN representation
+        extra_blocks: if provided, extra operations will
+            be performed. It is expected to take the fpn features, the original
+            features and the names of the original features as input, and returns
+            a new list of feature maps and their corresponding names
+
+    Examples::
+
+        >>> m = FeaturePyramidNetwork(2, [10, 20, 30], 5)
+        >>> # get some dummy data
+        >>> x = OrderedDict()
+        >>> x['feat0'] = torch.rand(1, 10, 64, 64)
+        >>> x['feat2'] = torch.rand(1, 20, 16, 16)
+        >>> x['feat3'] = torch.rand(1, 30, 8, 8)
+        >>> # compute the FPN on top of x
+        >>> output = m(x)
+        >>> print([(k, v.shape) for k, v in output.items()])
+        >>> # returns
+        >>>   [('feat0', torch.Size([1, 5, 64, 64])),
+        >>>    ('feat2', torch.Size([1, 5, 16, 16])),
+        >>>    ('feat3', torch.Size([1, 5, 8, 8]))]
+
+    """
+
+    def __init__(
+        self,
+        spatial_dims: int,
+        in_channels_list: List[int],
+        out_channels: int,
+        extra_blocks: Optional[ExtraFPNBlock] = None,
+    ):
+        super().__init__()
+
+        conv_type: Callable = Conv[Conv.CONV, spatial_dims]
+
+        self.inner_blocks = nn.ModuleList()
+        self.layer_blocks = nn.ModuleList()
+        for in_channels in in_channels_list:
+            if in_channels == 0:
+                raise ValueError("in_channels=0 is currently not supported")
+            inner_block_module = conv_type(in_channels, out_channels, 1)
+            layer_block_module = conv_type(out_channels, out_channels, 3, padding=1)
+            self.inner_blocks.append(inner_block_module)
+            self.layer_blocks.append(layer_block_module)
+
+        # initialize parameters now to avoid modifying the initialization of top_blocks
+        conv_type_: Type[nn.Module] = Conv[Conv.CONV, spatial_dims]
+        for m in self.modules():
+            if isinstance(m, conv_type_):
+                nn.init.kaiming_uniform_(m.weight, a=1)
+                nn.init.constant_(m.bias, 0.0)  # type: ignore
+
+        if extra_blocks is not None:
+            if not isinstance(extra_blocks, ExtraFPNBlock):
+                raise AssertionError
+        self.extra_blocks = extra_blocks
+
+    def get_result_from_inner_blocks(self, x: Tensor, idx: int) -> Tensor:
+        """
+        This is equivalent to self.inner_blocks[idx](x),
+        but torchscript doesn't support this yet
+        """
+        num_blocks = len(self.inner_blocks)
+        if idx < 0:
+            idx += num_blocks
+        out = x
+        for i, module in enumerate(self.inner_blocks):
+            if i == idx:
+                out = module(x)
+        return out
+
+    def get_result_from_layer_blocks(self, x: Tensor, idx: int) -> Tensor:
+        """
+        This is equivalent to self.layer_blocks[idx](x),
+        but torchscript doesn't support this yet
+        """
+        num_blocks = len(self.layer_blocks)
+        if idx < 0:
+            idx += num_blocks
+        out = x
+        for i, module in enumerate(self.layer_blocks):
+            if i == idx:
+                out = module(x)
+        return out
+
+    def forward(self, x: Dict[str, Tensor]) -> Dict[str, Tensor]:
+        """
+        Computes the FPN for a set of feature maps.
+
+        Args:
+            x: feature maps for each feature level.
+
+        Returns:
+            feature maps after FPN layers. They are ordered from highest resolution first.
+        """
+        # unpack OrderedDict into two lists for easier handling
+        names = list(x.keys())
+        x_values: List[Tensor] = list(x.values())
+
+        last_inner = self.get_result_from_inner_blocks(x_values[-1], -1)
+        results = []
+        results.append(self.get_result_from_layer_blocks(last_inner, -1))
+
+        for idx in range(len(x_values) - 2, -1, -1):
+            inner_lateral = self.get_result_from_inner_blocks(x_values[idx], idx)
+            feat_shape = inner_lateral.shape[2:]
+            inner_top_down = F.interpolate(last_inner, size=feat_shape, mode="nearest")
+            last_inner = inner_lateral + inner_top_down
+            results.insert(0, self.get_result_from_layer_blocks(last_inner, idx))
+
+        if self.extra_blocks is not None:
+            results, names = self.extra_blocks(results, x_values, names)
+
+        # make it back an OrderedDict
+        out = OrderedDict([(k, v) for k, v in zip(names, results)])
+
+        return out

my_container_sandbox/workspace/anaconda3/lib/python3.8/site-packages/monai/networks/blocks/localnet_block.py

@@ -0,0 +1,286 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional, Sequence, Tuple, Type, Union
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+from monai.networks.blocks import Convolution
+from monai.networks.layers import same_padding
+from monai.networks.layers.factories import Conv, Norm, Pool
+
+
+def get_conv_block(
+    spatial_dims: int,
+    in_channels: int,
+    out_channels: int,
+    kernel_size: Union[Sequence[int], int] = 3,
+    act: Optional[Union[Tuple, str]] = "RELU",
+    norm: Optional[Union[Tuple, str]] = "BATCH",
+) -> nn.Module:
+    padding = same_padding(kernel_size)
+    mod: nn.Module = Convolution(
+        spatial_dims,
+        in_channels,
+        out_channels,
+        kernel_size=kernel_size,
+        act=act,
+        norm=norm,
+        bias=False,
+        conv_only=False,
+        padding=padding,
+    )
+    return mod
+
+
+def get_conv_layer(
+    spatial_dims: int, in_channels: int, out_channels: int, kernel_size: Union[Sequence[int], int] = 3
+) -> nn.Module:
+    padding = same_padding(kernel_size)
+    mod: nn.Module = Convolution(
+        spatial_dims, in_channels, out_channels, kernel_size=kernel_size, bias=False, conv_only=True, padding=padding
+    )
+    return mod
+
+
+def get_deconv_block(spatial_dims: int, in_channels: int, out_channels: int) -> nn.Module:
+    mod: nn.Module = Convolution(
+        spatial_dims=spatial_dims,
+        in_channels=in_channels,
+        out_channels=out_channels,
+        strides=2,
+        act="RELU",
+        norm="BATCH",
+        bias=False,
+        is_transposed=True,
+        padding=1,
+        output_padding=1,
+    )
+    return mod
+
+
+class ResidualBlock(nn.Module):
+    def __init__(
+        self, spatial_dims: int, in_channels: int, out_channels: int, kernel_size: Union[Sequence[int], int]
+    ) -> None:
+        super().__init__()
+        if in_channels != out_channels:
+            raise ValueError(
+                f"expecting in_channels == out_channels, " f"got in_channels={in_channels}, out_channels={out_channels}"
+            )
+        self.conv_block = get_conv_block(
+            spatial_dims=spatial_dims, in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size
+        )
+        self.conv = get_conv_layer(
+            spatial_dims=spatial_dims, in_channels=out_channels, out_channels=out_channels, kernel_size=kernel_size
+        )
+        self.norm = Norm[Norm.BATCH, spatial_dims](out_channels)
+        self.relu = nn.ReLU()
+
+    def forward(self, x) -> torch.Tensor:
+        out: torch.Tensor = self.relu(self.norm(self.conv(self.conv_block(x))) + x)
+        return out
+
+
+class LocalNetResidualBlock(nn.Module):
+    def __init__(self, spatial_dims: int, in_channels: int, out_channels: int) -> None:
+        super().__init__()
+        if in_channels != out_channels:
+            raise ValueError(
+                f"expecting in_channels == out_channels, " f"got in_channels={in_channels}, out_channels={out_channels}"
+            )
+        self.conv_layer = get_conv_layer(spatial_dims=spatial_dims, in_channels=in_channels, out_channels=out_channels)
+        self.norm = Norm[Norm.BATCH, spatial_dims](out_channels)
+        self.relu = nn.ReLU()
+
+    def forward(self, x, mid) -> torch.Tensor:
+        out: torch.Tensor = self.relu(self.norm(self.conv_layer(x)) + mid)
+        return out
+
+
+class LocalNetDownSampleBlock(nn.Module):
+    """
+    A down-sample module that can be used for LocalNet, based on:
+    `Weakly-supervised convolutional neural networks for multimodal image registration
+    <https://doi.org/10.1016/j.media.2018.07.002>`_.
+    `Label-driven weakly-supervised learning for multimodal deformable image registration
+    <https://arxiv.org/abs/1711.01666>`_.
+
+    Adapted from:
+        DeepReg (https://github.com/DeepRegNet/DeepReg)
+    """
+
+    def __init__(
+        self, spatial_dims: int, in_channels: int, out_channels: int, kernel_size: Union[Sequence[int], int]
+    ) -> None:
+        """
+        Args:
+            spatial_dims: number of spatial dimensions.
+            in_channels: number of input channels.
+            out_channels: number of output channels.
+            kernel_size: convolution kernel size.
+        Raises:
+            NotImplementedError: when ``kernel_size`` is even
+        """
+        super().__init__()
+        self.conv_block = get_conv_block(
+            spatial_dims=spatial_dims, in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size
+        )
+        self.residual_block = ResidualBlock(
+            spatial_dims=spatial_dims, in_channels=out_channels, out_channels=out_channels, kernel_size=kernel_size
+        )
+        self.max_pool = Pool[Pool.MAX, spatial_dims](kernel_size=2)
+
+    def forward(self, x) -> Tuple[torch.Tensor, torch.Tensor]:
+        """
+        Halves the spatial dimensions.
+        A tuple of (x, mid) is returned:
+
+            -  x is the downsample result, in shape (batch, ``out_channels``, insize_1 / 2, insize_2 / 2, [insize_3 / 2]),
+            -  mid is the mid-level feature, in shape (batch, ``out_channels``, insize_1, insize_2, [insize_3])
+
+        Args:
+            x: Tensor in shape (batch, ``in_channels``, insize_1, insize_2, [insize_3])
+
+        Raises:
+            ValueError: when input spatial dimensions are not even.
+        """
+        for i in x.shape[2:]:
+            if i % 2 != 0:
+                raise ValueError("expecting x spatial dimensions be even, " f"got x of shape {x.shape}")
+        x = self.conv_block(x)
+        mid = self.residual_block(x)
+        x = self.max_pool(mid)
+        return x, mid
+
+
+class LocalNetUpSampleBlock(nn.Module):
+    """
+    A up-sample module that can be used for LocalNet, based on:
+    `Weakly-supervised convolutional neural networks for multimodal image registration
+    <https://doi.org/10.1016/j.media.2018.07.002>`_.
+    `Label-driven weakly-supervised learning for multimodal deformable image registration
+    <https://arxiv.org/abs/1711.01666>`_.
+
+    Adapted from:
+        DeepReg (https://github.com/DeepRegNet/DeepReg)
+    """
+
+    def __init__(self, spatial_dims: int, in_channels: int, out_channels: int) -> None:
+        """
+        Args:
+            spatial_dims: number of spatial dimensions.
+            in_channels: number of input channels.
+            out_channels: number of output channels.
+        Raises:
+            ValueError: when ``in_channels != 2 * out_channels``
+        """
+        super().__init__()
+        self.deconv_block = get_deconv_block(
+            spatial_dims=spatial_dims, in_channels=in_channels, out_channels=out_channels
+        )
+        self.conv_block = get_conv_block(spatial_dims=spatial_dims, in_channels=out_channels, out_channels=out_channels)
+        self.residual_block = LocalNetResidualBlock(
+            spatial_dims=spatial_dims, in_channels=out_channels, out_channels=out_channels
+        )
+        if in_channels / out_channels != 2:
+            raise ValueError(
+                f"expecting in_channels == 2 * out_channels, "
+                f"got in_channels={in_channels}, out_channels={out_channels}"
+            )
+        self.out_channels = out_channels
+
+    def addictive_upsampling(self, x, mid) -> torch.Tensor:
+        x = F.interpolate(x, mid.shape[2:])
+        # [(batch, out_channels, ...), (batch, out_channels, ...)]
+        x = x.split(split_size=int(self.out_channels), dim=1)
+        # (batch, out_channels, ...)
+        out: torch.Tensor = torch.sum(torch.stack(x, dim=-1), dim=-1)
+        return out
+
+    def forward(self, x, mid) -> torch.Tensor:
+        """
+        Halves the channel and doubles the spatial dimensions.
+
+        Args:
+            x: feature to be up-sampled, in shape (batch, ``in_channels``, insize_1, insize_2, [insize_3])
+            mid: mid-level feature saved during down-sampling,
+                in shape (batch, ``out_channels``, midsize_1, midsize_2, [midsize_3])
+
+        Raises:
+            ValueError: when ``midsize != insize * 2``
+        """
+        for i, j in zip(x.shape[2:], mid.shape[2:]):
+            if j != 2 * i:
+                raise ValueError(
+                    "expecting mid spatial dimensions be exactly the double of x spatial dimensions, "
+                    f"got x of shape {x.shape}, mid of shape {mid.shape}"
+                )
+        h0 = self.deconv_block(x) + self.addictive_upsampling(x, mid)
+        r1 = h0 + mid
+        r2 = self.conv_block(h0)
+        out: torch.Tensor = self.residual_block(r2, r1)
+        return out
+
+
+class LocalNetFeatureExtractorBlock(nn.Module):
+    """
+    A feature-extraction module that can be used for LocalNet, based on:
+    `Weakly-supervised convolutional neural networks for multimodal image registration
+    <https://doi.org/10.1016/j.media.2018.07.002>`_.
+    `Label-driven weakly-supervised learning for multimodal deformable image registration
+    <https://arxiv.org/abs/1711.01666>`_.
+
+    Adapted from:
+        DeepReg (https://github.com/DeepRegNet/DeepReg)
+    """
+
+    def __init__(
+        self,
+        spatial_dims: int,
+        in_channels: int,
+        out_channels: int,
+        act: Optional[Union[Tuple, str]] = "RELU",
+        initializer: str = "kaiming_uniform",
+    ) -> None:
+        """
+        Args:
+        spatial_dims: number of spatial dimensions.
+        in_channels: number of input channels.
+        out_channels: number of output channels.
+        act: activation type and arguments. Defaults to ReLU.
+        kernel_initializer: kernel initializer. Defaults to None.
+        """
+        super().__init__()
+        self.conv_block = get_conv_block(
+            spatial_dims=spatial_dims, in_channels=in_channels, out_channels=out_channels, act=act, norm=None
+        )
+        conv_type: Type[Union[nn.Conv1d, nn.Conv2d, nn.Conv3d]] = Conv[Conv.CONV, spatial_dims]
+        for m in self.conv_block.modules():
+            if isinstance(m, conv_type):
+                if initializer == "kaiming_uniform":
+                    nn.init.kaiming_normal_(torch.as_tensor(m.weight))
+                elif initializer == "zeros":
+                    nn.init.zeros_(torch.as_tensor(m.weight))
+                else:
+                    raise ValueError(
+                        f"initializer {initializer} is not supported, " "currently supporting kaiming_uniform and zeros"
+                    )
+
+    def forward(self, x) -> torch.Tensor:
+        """
+        Args:
+            x: Tensor in shape (batch, ``in_channels``, insize_1, insize_2, [insize_3])
+        """
+        out: torch.Tensor = self.conv_block(x)
+        return out