from megatron.model.module import MegatronModule
from megatron.model.vision.utils import resize

class SetrSegmentationHead(MegatronModule):
    def __init__(self, hidden_size, num_classes):
        super(SetrSegmentationHead, self).__init__()
        args = get_args()
        self.hidden_size = hidden_size
        self.num_classes = num_classes
        self.img_h = args.img_h
        self.img_w = args.img_w
        self.patch_dim = args.patch_dim

        self.layernorm = LayerNorm(hidden_size, eps=args.layernorm_epsilon)
        self.conv_0 = torch.nn.Conv2d(hidden_size, hidden_size,
                                      1, 1, bias=False)
        self.norm_0 = apex.parallel.SyncBatchNorm(hidden_size)
        self.conv_1 = torch.nn.Conv2d(hidden_size, num_classes, 1, 1)

    def to_2D(self, x):
        n, hw, c = x.shape
        h = self.img_h // self.patch_dim
        w = self.img_w // self.patch_dim
        assert(hw == h * w)
        x = x.transpose(1, 2).reshape(n, c, h, w)
        return x

    def forward(self, hidden_states):
        # [b c h w]
        hidden_states = self.layernorm(hidden_states)
        hidden_states = self.to_2D(hidden_states)

        hidden_states = self.conv_0(hidden_states)
        hidden_states = self.norm_0(hidden_states)
        hidden_states = torch.tanh(hidden_states)
        hidden_states = self.conv_1(hidden_states)

        # [b c h w]
        result = F.interpolate(hidden_states,
                               size=(self.img_h, self.img_w),
                               mode='bilinear')

        return result

class MLP(torch.nn.Module):


forward

            self.in_channels

        self.linear_c4 = MLP(input_dim=c4_in_channels,
                             embed_dim=self.embedding_dim)
        self.linear_c3 = MLP(input_dim=c3_in_channels,
                             embed_dim=self.embedding_dim)
        self.linear_c2 = MLP(input_dim=c2_in_channels,
                             embed_dim=self.embedding_dim)
        self.linear_c1 = MLP(input_dim=c1_in_channels,
                             embed_dim=self.embedding_dim)

        self.conv_fuse = torch.nn.Conv2d(self.embedding_dim*4,
                                         self.embedding_dim, 1, 1)
        self.norm = apex.parallel.SyncBatchNorm(self.embedding_dim)

        self.dropout = torch.nn.Dropout2d(self.dropout_ratio)
        self.linear_pred = torch.nn.Conv2d(self.embedding_dim,
                                           self.num_classes,
                                           kernel_size=1)

    def forward(self, inputs):
        c1, c2, c3, c4 = inputs

        ############## MLP decoder on C1-C4 ###########
        n, _, h, w = c4.shape

        _c4 = self.linear_c4(c4).permute(0, 2, 1).reshape(n, -1, c4.shape[2], c4.shape[3])
        _c4 = resize(_c4, size=c1.size()[2:], mode='bilinear', align_corners=False)

        _c3 = self.linear_c3(c3).permute(0, 2, 1).reshape(n, -1, c3.shape[2], c3.shape[3])
        _c3 = resize(_c3, size=c1.size()[2:], mode='bilinear', align_corners=False)

        _c2 = self.linear_c2(c2).permute(0, 2, 1).reshape(n, -1, c2.shape[2], c2.shape[3])
        _c2 = resize(_c2, size=c1.size()[2:], mode='bilinear', align_corners=False)

        _c1 = self.linear_c1(c1).permute(0, 2, 1).reshape(n, -1, c1.shape[2], c1.shape[3])

        _c = self.conv_fuse(torch.cat([_c4, _c3, _c2, _c1], dim=1))
        x = self.norm(_c)
        x = F.relu(x, inplace=True)
        x = self.dropout(x)
        x = self.linear_pred(x)

        return x


tasks.vision.segmentation.cityscapes

# BSD 3-Clause License
#
# Copyright (c) Soumith Chintala 2016,
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of the copyright holder nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# code taken from
# https://github.com/pytorch/vision/blob/main/torchvision/datasets/cityscapes.py
# modified it to change max label index from 255 to 19 (num_classes)

import torch
import json
import os
from collections import namedtuple
from typing import Any, Callable, Dict, List, Optional, Union, Tuple
import numpy as np
from torchvision.datasets.utils import extract_archive, verify_str_arg, iterable_to_str
from torchvision.datasets import VisionDataset
from PIL import Image
from megatron import print_rank_0

class Cityscapes(VisionDataset):
    """`Cityscapes <http://www.cityscapes-dataset.com/>`_ Dataset.
    Args:
        root (string): Root directory of dataset where directory ``leftImg8bit``
            and ``gtFine`` or ``gtCoarse`` are located.
        split (string, optional): The image split to use, ``train``, ``test`` or ``val`` if mode="fine"
            otherwise ``train``, ``train_extra`` or ``val``
        mode (string, optional): The quality mode to use, ``fine`` or ``coarse``
        target_type (string or list, optional): Type of target to use, ``instance``, ``semantic``, ``polygon``
            or ``color``. Can also be a list to output a tuple with all specified target types.
        transform (callable, optional): A function/transform that takes in a PIL image
            and returns a transformed version. E.g, ``transforms.RandomCrop``
        target_transform (callable, optional): A function/transform that takes in the
            target and transforms it.
        transforms (callable, optional): A function/transform that takes input sample and its target as entry
            and returns a transformed version.
    Examples:
        Get semantic segmentation target
        .. code-block:: python
            dataset = Cityscapes('./data/cityscapes', split='train', mode='fine',
                                 target_type='semantic')
            img, smnt = dataset[0]
        Get multiple targets
        .. code-block:: python
            dataset = Cityscapes('./data/cityscapes', split='train', mode='fine',
                                 target_type=['instance', 'color', 'polygon'])
            img, (inst, col, poly) = dataset[0]
        Validate on the "coarse" set
        .. code-block:: python
            dataset = Cityscapes('./data/cityscapes', split='val', mode='coarse',
                                 target_type='semantic')
            img, smnt = dataset[0]
    """
    num_classes = 19
    ignore_index = 19
    color_table = torch.tensor(
        [[128, 64, 128],
         [244, 35, 232],
         [70, 70, 70],
         [102, 102, 156],
         [190, 153, 153],
         [153, 153, 153],
         [250, 170, 30],
         [220, 220, 0],
         [107, 142, 35],
         [152, 251, 152],
         [70, 130, 180],
         [220, 20, 60],
         [255, 0, 0],
         [0, 0, 142],
         [0, 0, 70],
         [0, 60, 100],
         [0, 80, 100],
         [0, 0, 230],
         [119, 11, 32],
         [0, 0, 0]], dtype=torch.float, device='cuda')

    # Based on https://github.com/mcordts/cityscapesScripts
    CityscapesClass = namedtuple('CityscapesClass', ['name', 'id', 'train_id',
        'category', 'category_id', 'has_instances', 'ignore_in_eval', 'color'])

    classes = [
        CityscapesClass('unlabeled', 0, 19, 'void', 0, False, True, (0, 0, 0)),
        CityscapesClass('ego vehicle', 1, 19, 'void', 0, False, True, (0, 0, 0)),
        CityscapesClass('rectification border', 2, 19, 'void', 0, False, True, (0, 0, 0)),
        CityscapesClass('out of roi', 3, 19, 'void', 0, False, True, (0, 0, 0)),
        CityscapesClass('static', 4, 19, 'void', 0, False, True, (0, 0, 0)),
        CityscapesClass('dynamic', 5, 19, 'void', 0, False, True, (111, 74, 0)),
        CityscapesClass('ground', 6, 19, 'void', 0, False, True, (81, 0, 81)),
        CityscapesClass('road', 7, 0, 'flat', 1, False, False, (128, 64, 128)),
        CityscapesClass('sidewalk', 8, 1, 'flat', 1, False, False, (244, 35, 232)),
        CityscapesClass('parking', 9, 19, 'flat', 1, False, True, (250, 170, 160)),
        CityscapesClass('rail track', 10, 19, 'flat', 1, False, True, (230, 150, 140)),
        CityscapesClass('building', 11, 2, 'construction', 2, False, False, (70, 70, 70)),
        CityscapesClass('wall', 12, 3, 'construction', 2, False, False, (102, 102, 156)),
        CityscapesClass('fence', 13, 4, 'construction', 2, False, False, (190, 153, 153)),
        CityscapesClass('guard rail', 14, 19, 'construction', 2, False, True, (180, 165, 180)),
        CityscapesClass('bridge', 15, 19, 'construction', 2, False, True, (150, 100, 100)),
        CityscapesClass('tunnel', 16, 19, 'construction', 2, False, True, (150, 120, 90)),
        CityscapesClass('pole', 17, 5, 'object', 3, False, False, (153, 153, 153)),
        CityscapesClass('polegroup', 18, 19, 'object', 3, False, True, (153, 153, 153)),
        CityscapesClass('traffic light', 19, 6, 'object', 3, False, False, (250, 170, 30)),
        CityscapesClass('traffic sign', 20, 7, 'object', 3, False, False, (220, 220, 0)),
        CityscapesClass('vegetation', 21, 8, 'nature', 4, False, False, (107, 142, 35)),
        CityscapesClass('terrain', 22, 9, 'nature', 4, False, False, (152, 251, 152)),
        CityscapesClass('sky', 23, 10, 'sky', 5, False, False, (70, 130, 180)),
        CityscapesClass('person', 24, 11, 'human', 6, True, False, (220, 20, 60)),
        CityscapesClass('rider', 25, 12, 'human', 6, True, False, (255, 0, 0)),
        CityscapesClass('car', 26, 13, 'vehicle', 7, True, False, (0, 0, 142)),
        CityscapesClass('truck', 27, 14, 'vehicle', 7, True, False, (0, 0, 70)),
        CityscapesClass('bus', 28, 15, 'vehicle', 7, True, False, (0, 60, 100)),
        CityscapesClass('caravan', 29, 19, 'vehicle', 7, True, True, (0, 0, 90)),
        CityscapesClass('trailer', 30, 19, 'vehicle', 7, True, True, (0, 0, 110)),
        CityscapesClass('train', 31, 16, 'vehicle', 7, True, False, (0, 80, 100)),
        CityscapesClass('motorcycle', 32, 17, 'vehicle', 7, True, False, (0, 0, 230)),
        CityscapesClass('bicycle', 33, 18, 'vehicle', 7, True, False, (119, 11, 32)),
        CityscapesClass('license plate', -1, -1, 'vehicle', 7, False, True, (0, 0, 142)),
    ]

    # label2trainid
    label2trainid   = { label.id  : label.train_id for label in classes}

    def __init__(
            self,
            root: str,
            split: str = "train",
            mode: str = "fine",
            resolution: int = 1024,
            transform: Optional[Callable] = None,
            target_transform: Optional[Callable] = None,
            transforms: Optional[Callable] = None,
    ) -> None:
        super(Cityscapes, self).__init__(root, transforms, transform, target_transform)
        self.mode = 'gtFine' if mode == 'fine' else 'gtCoarse'
        self.images_dir = os.path.join(self.root, 'leftImg8bit_trainvaltest/leftImg8bit', split)
        self.targets_dir = os.path.join(self.root, 'gtFine_trainvaltest/gtFine', split)
        self.split = split
        self.resolution = resolution
        self.images = []
        self.targets = []

        for city in sorted(os.listdir(self.images_dir)):
            img_dir = os.path.join(self.images_dir, city)
            target_dir = os.path.join(self.targets_dir, city)
            for file_name in os.listdir(img_dir):
                target_name = '{}_{}_labelIds.png'.format(file_name.split('_leftImg8bit')[0], self.mode)
                self.images.append(os.path.join(img_dir, file_name))
                self.targets.append(os.path.join(target_dir, target_name))

    def __getitem__(self, index: int) -> Tuple[Any, Any]:
        """
        Args:
            index (int): Index
        Returns:
            tuple: (image, target) where target is a tuple of all target types if target_type is a list with more
            than one item. Otherwise target is a json object if target_type="polygon", else the image segmentation.
        """
        image = Image.open(self.images[index]).convert('RGB')

        target = Image.open(self.targets[index])
        target = np.array(target)

        target_copy = target.copy()
        for k, v in Cityscapes.label2trainid.items():
            binary_target = (target == k)
            target_copy[binary_target] = v
        target = target_copy

        target = Image.fromarray(target.astype(np.uint8))

        if self.transforms is not None:
            image, target = self.transforms(image, target)

        return image, target

    def __len__(self) -> int:
        # len(self.images)
        return len(self.images)


Cityscapes

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# code taken from
# https://github.com/pytorch/vision/blob/main/torchvision/datasets/cityscapes.py
# modified it to change max label index from 255 to 19 (num_classes)

import torch
import json
import os
from collections import namedtuple
from typing import Any, Callable, Dict, List, Optional, Union, Tuple
import numpy as np
from torchvision.datasets.utils import extract_archive, verify_str_arg, iterable_to_str
from torchvision.datasets import VisionDataset
from PIL import Image
from megatron import print_rank_0

class Cityscapes(VisionDataset):
    """`Cityscapes <http://www.cityscapes-dataset.com/>`_ Dataset.
    Args:
        root (string): Root directory of dataset where directory ``leftImg8bit``
            and ``gtFine`` or ``gtCoarse`` are located.
        split (string, optional): The image split to use, ``train``, ``test`` or ``val`` if mode="fine"
            otherwise ``train``, ``train_extra`` or ``val``
        mode (string, optional): The quality mode to use, ``fine`` or ``coarse``
        target_type (string or list, optional): Type of target to use, ``instance``, ``semantic``, ``polygon``
            or ``color``. Can also be a list to output a tuple with all specified target types.
        transform (callable, optional): A function/transform that takes in a PIL image
            and returns a transformed version. E.g, ``transforms.RandomCrop``
        target_transform (callable, optional): A function/transform that takes in the
            target and transforms it.
        transforms (callable, optional): A function/transform that takes input sample and its target as entry
            and returns a transformed version.
    Examples:
        Get semantic segmentation target
        .. code-block:: python
            dataset = Cityscapes('./data/cityscapes', split='train', mode='fine',
                                 target_type='semantic')
            img, smnt = dataset[0]
        Get multiple targets
        .. code-block:: python
            dataset = Cityscapes('./data/cityscapes', split='train', mode='fine',
                                 target_type=['instance', 'color', 'polygon'])
            img, (inst, col, poly) = dataset[0]
        Validate on the "coarse" set
        .. code-block:: python
            dataset = Cityscapes('./data/cityscapes', split='val', mode='coarse',
                                 target_type='semantic')
            img, smnt = dataset[0]
    """
    num_classes = 19
    ignore_index = 19
    color_table = torch.tensor(
        [[128, 64, 128],
         [244, 35, 232],
         [70, 70, 70],
         [102, 102, 156],
         [190, 153, 153],
         [153, 153, 153],
         [250, 170, 30],
         [220, 220, 0],
         [107, 142, 35],
         [152, 251, 152],
         [70, 130, 180],
         [220, 20, 60],
         [255, 0, 0],
         [0, 0, 142],
         [0, 0, 70],
         [0, 60, 100],
         [0, 80, 100],
         [0, 0, 230],
         [119, 11, 32],
         [0, 0, 0]], dtype=torch.float, device='cuda')

    # Based on https://github.com/mcordts/cityscapesScripts
    CityscapesClass = namedtuple('CityscapesClass', ['name', 'id', 'train_id',
        'category', 'category_id', 'has_instances', 'ignore_in_eval', 'color'])

    classes = [
        CityscapesClass('unlabeled', 0, 19, 'void', 0, False, True, (0, 0, 0)),
        CityscapesClass('ego vehicle', 1, 19, 'void', 0, False, True, (0, 0, 0)),
        CityscapesClass('rectification border', 2, 19, 'void', 0, False, True, (0, 0, 0)),
        CityscapesClass('out of roi', 3, 19, 'void', 0, False, True, (0, 0, 0)),
        CityscapesClass('static', 4, 19, 'void', 0, False, True, (0, 0, 0)),
        CityscapesClass('dynamic', 5, 19, 'void', 0, False, True, (111, 74, 0)),
        CityscapesClass('ground', 6, 19, 'void', 0, False, True, (81, 0, 81)),
        CityscapesClass('road', 7, 0, 'flat', 1, False, False, (128, 64, 128)),
        CityscapesClass('sidewalk', 8, 1, 'flat', 1, False, False, (244, 35, 232)),
        CityscapesClass('parking', 9, 19, 'flat', 1, False, True, (250, 170, 160)),
        CityscapesClass('rail track', 10, 19, 'flat', 1, False, True, (230, 150, 140)),
        CityscapesClass('building', 11, 2, 'construction', 2, False, False, (70, 70, 70)),
        CityscapesClass('wall', 12, 3, 'construction', 2, False, False, (102, 102, 156)),
        CityscapesClass('fence', 13, 4, 'construction', 2, False, False, (190, 153, 153)),
        CityscapesClass('guard rail', 14, 19, 'construction', 2, False, True, (180, 165, 180)),
        CityscapesClass('bridge', 15, 19, 'construction', 2, False, True, (150, 100, 100)),
        CityscapesClass('tunnel', 16, 19, 'construction', 2, False, True, (150, 120, 90)),
        CityscapesClass('pole', 17, 5, 'object', 3, False, False, (153, 153, 153)),
        CityscapesClass('polegroup', 18, 19, 'object', 3, False, True, (153, 153, 153)),
        CityscapesClass('traffic light', 19, 6, 'object', 3, False, False, (250, 170, 30)),
        CityscapesClass('traffic sign', 20, 7, 'object', 3, False, False, (220, 220, 0)),
        CityscapesClass('vegetation', 21, 8, 'nature', 4, False, False, (107, 142, 35)),
        CityscapesClass('terrain', 22, 9, 'nature', 4, False, False, (152, 251, 152)),
        CityscapesClass('sky', 23, 10, 'sky', 5, False, False, (70, 130, 180)),
        CityscapesClass('person', 24, 11, 'human', 6, True, False, (220, 20, 60)),
        CityscapesClass('rider', 25, 12, 'human', 6, True, False, (255, 0, 0)),
        CityscapesClass('car', 26, 13, 'vehicle', 7, True, False, (0, 0, 142)),
        CityscapesClass('truck', 27, 14, 'vehicle', 7, True, False, (0, 0, 70)),
        CityscapesClass('bus', 28, 15, 'vehicle', 7, True, False, (0, 60, 100)),
        CityscapesClass('caravan', 29, 19, 'vehicle', 7, True, True, (0, 0, 90)),
        CityscapesClass('trailer', 30, 19, 'vehicle', 7, True, True, (0, 0, 110)),
        CityscapesClass('train', 31, 16, 'vehicle', 7, True, False, (0, 80, 100)),
        CityscapesClass('motorcycle', 32, 17, 'vehicle', 7, True, False, (0, 0, 230)),
        CityscapesClass('bicycle', 33, 18, 'vehicle', 7, True, False, (119, 11, 32)),
        CityscapesClass('license plate', -1, -1, 'vehicle', 7, False, True, (0, 0, 142)),
    ]

    # label2trainid
    label2trainid   = { label.id  : label.train_id for label in classes}

    def __init__(
            self,
            root: str,
            split: str = "train",
            mode: str = "fine",
            resolution: int = 1024,
            transform: Optional[Callable] = None,
            target_transform: Optional[Callable] = None,
            transforms: Optional[Callable] = None,
    ) -> None:
        super(Cityscapes, self).__init__(root, transforms, transform, target_transform)
        self.mode = 'gtFine' if mode == 'fine' else 'gtCoarse'
        self.images_dir = os.path.join(self.root, 'leftImg8bit_trainvaltest/leftImg8bit', split)
        self.targets_dir = os.path.join(self.root, 'gtFine_trainvaltest/gtFine', split)
        self.split = split
        self.resolution = resolution
        self.images = []
        self.targets = []

        for city in sorted(os.listdir(self.images_dir)):
            img_dir = os.path.join(self.images_dir, city)
            target_dir = os.path.join(self.targets_dir, city)
            for file_name in os.listdir(img_dir):
                target_name = '{}_{}_labelIds.png'.format(file_name.split('_leftImg8bit')[0], self.mode)
                self.images.append(os.path.join(img_dir, file_name))
                self.targets.append(os.path.join(target_dir, target_name))

    def __getitem__(self, index: int) -> Tuple[Any, Any]:
        """
        Args:
            index (int): Index
        Returns:
            tuple: (image, target) where target is a tuple of all target types if target_type is a list with more
            than one item. Otherwise target is a json object if target_type="polygon", else the image segmentation.
        """
        image = Image.open(self.images[index]).convert('RGB')

        target = Image.open(self.targets[index])
        target = np.array(target)

        target_copy = target.copy()
        for k, v in Cityscapes.label2trainid.items():
            binary_target = (target == k)
            target_copy[binary_target] = v
        target = target_copy

        target = Image.fromarray(target.astype(np.uint8))

        if self.transforms is not None:
            image, target = self.transforms(image, target)

        return image, target

    def __len__(self) -> int:
        # len(self.images)
        return len(self.images)


        CityscapesClass('traffic sign', 20, 7, 'object', 3, False, False, (220, 220, 0)),
        CityscapesClass('vegetation', 21, 8, 'nature', 4, False, False, (107, 142, 35)),
        CityscapesClass('terrain', 22, 9, 'nature', 4, False, False, (152, 251, 152)),
        CityscapesClass('sky', 23, 10, 'sky', 5, False, False, (70, 130, 180)),
        CityscapesClass('person', 24, 11, 'human', 6, True, False, (220, 20, 60)),
        CityscapesClass('rider', 25, 12, 'human', 6, True, False, (255, 0, 0)),
        CityscapesClass('car', 26, 13, 'vehicle', 7, True, False, (0, 0, 142)),
        CityscapesClass('truck', 27, 14, 'vehicle', 7, True, False, (0, 0, 70)),
        CityscapesClass('bus', 28, 15, 'vehicle', 7, True, False, (0, 60, 100)),
        CityscapesClass('caravan', 29, 19, 'vehicle', 7, True, True, (0, 0, 90)),
        CityscapesClass('trailer', 30, 19, 'vehicle', 7, True, True, (0, 0, 110)),
        CityscapesClass('train', 31, 16, 'vehicle', 7, True, False, (0, 80, 100)),
        CityscapesClass('motorcycle', 32, 17, 'vehicle', 7, True, False, (0, 0, 230)),
        CityscapesClass('bicycle', 33, 18, 'vehicle', 7, True, False, (119, 11, 32)),
        CityscapesClass('license plate', -1, -1, 'vehicle', 7, False, True, (0, 0, 142)),
    ]

    # label2trainid
    label2trainid   = { label.id  : label.train_id for label in classes}

    def __init__(
            self,
            root: str,
            split: str = "train",
            mode: str = "fine",
            resolution: int = 1024,
            transform: Optional[Callable] = None,
            target_transform: Optional[Callable] = None,
            transforms: Optional[Callable] = None,
    ) -> None:
        super(Cityscapes, self).__init__(root, transforms, transform, target_transform)
        self.mode = 'gtFine' if mode == 'fine' else 'gtCoarse'
        self.images_dir = os.path.join(self.root, 'leftImg8bit_trainvaltest/leftImg8bit', split)
        self.targets_dir = os.path.join(self.root, 'gtFine_trainvaltest/gtFine', split)
        self.split = split
        self.resolution = resolution
        self.images = []
        self.targets = []

        for city in sorted(os.listdir(self.images_dir)):
            img_dir = os.path.join(self.images_dir, city)
            target_dir = os.path.join(self.targets_dir, city)
            for file_name in os.listdir(img_dir):
                target_name = '{}_{}_labelIds.png'.format(file_name.split('_leftImg8bit')[0], self.mode)
                self.images.append(os.path.join(img_dir, file_name))
                self.targets.append(os.path.join(target_dir, target_name))

    def __getitem__(self, index: int) -> Tuple[Any, Any]:
        """
        Args:
            index (int): Index
        Returns:
            tuple: (image, target) where target is a tuple of all target types if target_type is a list with more
            than one item. Otherwise target is a json object if target_type="polygon", else the image segmentation.
        """
        image = Image.open(self.images[index]).convert('RGB')

        target = Image.open(self.targets[index])
        target = np.array(target)

        target_copy = target.copy()
        for k, v in Cityscapes.label2trainid.items():
            binary_target = (target == k)
            target_copy[binary_target] = v
        target = target_copy


            transforms: Optional[Callable] = None,
    ) -> None:
        super(Cityscapes, self).__init__(root, transforms, transform, target_transform)
        self.mode = 'gtFine' if mode == 'fine' else 'gtCoarse'
        self.images_dir = os.path.join(self.root, 'leftImg8bit_trainvaltest/leftImg8bit', split)
        self.targets_dir = os.path.join(self.root, 'gtFine_trainvaltest/gtFine', split)
        self.split = split
        self.resolution = resolution
        self.images = []
        self.targets = []

        for city in sorted(os.listdir(self.images_dir)):
            img_dir = os.path.join(self.images_dir, city)
            target_dir = os.path.join(self.targets_dir, city)
            for file_name in os.listdir(img_dir):
                target_name = '{}_{}_labelIds.png'.format(file_name.split('_leftImg8bit')[0], self.mode)
                self.images.append(os.path.join(img_dir, file_name))
                self.targets.append(os.path.join(target_dir, target_name))

    def __getitem__(self, index: int) -> Tuple[Any, Any]:
        """
        Args:
            index (int): Index
        Returns:
            tuple: (image, target) where target is a tuple of all target types if target_type is a list with more
            than one item. Otherwise target is a json object if target_type="polygon", else the image segmentation.
        """
        image = Image.open(self.images[index]).convert('RGB')

        target = Image.open(self.targets[index])
        target = np.array(target)

        target_copy = target.copy()
        for k, v in Cityscapes.label2trainid.items():
            binary_target = (target == k)
            target_copy[binary_target] = v
        target = target_copy

        target = Image.fromarray(target.astype(np.uint8))

        if self.transforms is not None:
            image, target = self.transforms(image, target)

        return image, target

    def __len__(self) -> int:
        # len(self.images)
        return len(self.images)


            than one item. Otherwise target is a json object if target_type="polygon", else the image segmentation.
        """
        image = Image.open(self.images[index]).convert('RGB')

        target = Image.open(self.targets[index])
        target = np.array(target)

        target_copy = target.copy()
        for k, v in Cityscapes.label2trainid.items():
            binary_target = (target == k)
            target_copy[binary_target] = v
        target = target_copy

        target = Image.fromarray(target.astype(np.uint8))

        if self.transforms is not None:
            image, target = self.transforms(image, target)

        return image, target

    def __len__(self) -> int:
        # len(self.images)
        return len(self.images)


# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
import math
import einops
import torch
import apex
import torch.nn.functional as F
from megatron import get_args
from megatron.model.module import MegatronModule
from megatron.model.vision.vit_backbone import VitBackbone, VitMlpHead
from megatron.model.vision.mit_backbone import mit_b3, mit_b5
from tasks.vision.segmentation.seg_heads import SetrSegmentationHead, SegformerSegmentationHead

class SetrSegmentationModel(MegatronModule):

    def __init__(self,
                 num_classes,
                 pre_process=True,
                 post_process=True):
        super(SetrSegmentationModel, self).__init__()
        args = get_args()
        assert post_process & pre_process
        self.hidden_size = args.hidden_size
        self.num_classes = num_classes
        self.backbone = VitBackbone(
            pre_process=pre_process,
            post_process=post_process,
            class_token=False,
            post_layer_norm=False,
            drop_path_rate=0.1
        )

        self.head = SetrSegmentationHead(
            self.hidden_size,
            self.num_classes
        )

    def set_input_tensor(self, input_tensor):
        """See megatron.model.transformer.set_input_tensor()"""
        pass

    def forward(self, input):
        # [b hw c]
        hidden_states = self.backbone(input)
        result_final = self.head(hidden_states)
        return result_final

class SegformerSegmentationModel(MegatronModule):

    def __init__(self,
                 num_classes,
                 pre_process=True,
                 post_process=True):
        super(SegformerSegmentationModel, self).__init__()
        args = get_args()
        self.hidden_size = args.hidden_size
        self.num_classes = num_classes
        self.pre_process = pre_process
        self.post_process = post_process

        self.backbone = mit_b5()
        self.head = SegformerSegmentationHead(
            feature_strides=[4, 8, 16, 32],
            in_channels=[64, 128, 320, 512],
            embedding_dim=768,
            dropout_ratio=0.1
        )

    def set_input_tensor(self, input_tensor):
        """See megatron.model.transformer.set_input_tensor()"""
        pass

    def forward(self, input):
        # [b hw c]
        hidden_states = self.backbone(input)
        hidden_states = self.head(hidden_states)
        return hidden_states


SetrSegmentationModel

# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
import math
import einops
import torch
import apex
import torch.nn.functional as F
from megatron import get_args
from megatron.model.module import MegatronModule
from megatron.model.vision.vit_backbone import VitBackbone, VitMlpHead
from megatron.model.vision.mit_backbone import mit_b3, mit_b5
from tasks.vision.segmentation.seg_heads import SetrSegmentationHead, SegformerSegmentationHead

class SetrSegmentationModel(MegatronModule):

    def __init__(self,
                 num_classes,
                 pre_process=True,
                 post_process=True):
        super(SetrSegmentationModel, self).__init__()
        args = get_args()
        assert post_process & pre_process
        self.hidden_size = args.hidden_size
        self.num_classes = num_classes
        self.backbone = VitBackbone(
            pre_process=pre_process,
            post_process=post_process,
            class_token=False,
            post_layer_norm=False,
            drop_path_rate=0.1
        )

        self.head = SetrSegmentationHead(
            self.hidden_size,
            self.num_classes
        )

    def set_input_tensor(self, input_tensor):
        """See megatron.model.transformer.set_input_tensor()"""
        pass

    def forward(self, input):
        # [b hw c]
        hidden_states = self.backbone(input)
        result_final = self.head(hidden_states)
        return result_final

class SegformerSegmentationModel(MegatronModule):

    def __init__(self,
                 num_classes,
                 pre_process=True,
                 post_process=True):
        super(SegformerSegmentationModel, self).__init__()
        args = get_args()
        self.hidden_size = args.hidden_size
        self.num_classes = num_classes
        self.pre_process = pre_process
        self.post_process = post_process

        self.backbone = mit_b5()
        self.head = SegformerSegmentationHead(
            feature_strides=[4, 8, 16, 32],
            in_channels=[64, 128, 320, 512],
            embedding_dim=768,


SegformerSegmentationModel

            post_layer_norm=False,
            drop_path_rate=0.1
        )

        self.head = SetrSegmentationHead(
            self.hidden_size,
            self.num_classes
        )

    def set_input_tensor(self, input_tensor):
        """See megatron.model.transformer.set_input_tensor()"""
        pass

    def forward(self, input):
        # [b hw c]
        hidden_states = self.backbone(input)
        result_final = self.head(hidden_states)
        return result_final

class SegformerSegmentationModel(MegatronModule):

    def __init__(self,
                 num_classes,
                 pre_process=True,
                 post_process=True):
        super(SegformerSegmentationModel, self).__init__()
        args = get_args()
        self.hidden_size = args.hidden_size
        self.num_classes = num_classes
        self.pre_process = pre_process
        self.post_process = post_process

        self.backbone = mit_b5()
        self.head = SegformerSegmentationHead(
            feature_strides=[4, 8, 16, 32],
            in_channels=[64, 128, 320, 512],
            embedding_dim=768,
            dropout_ratio=0.1
        )

    def set_input_tensor(self, input_tensor):
        """See megatron.model.transformer.set_input_tensor()"""
        pass

    def forward(self, input):
        # [b hw c]
        hidden_states = self.backbone(input)
        hidden_states = self.head(hidden_states)
        return hidden_states


        )

        self.head = SetrSegmentationHead(
            self.hidden_size,
            self.num_classes
        )

    def set_input_tensor(self, input_tensor):
        """See megatron.model.transformer.set_input_tensor()"""
        pass

    def forward(self, input):
        # [b hw c]
        hidden_states = self.backbone(input)
        result_final = self.head(hidden_states)
        return result_final

class SegformerSegmentationModel(MegatronModule):

    def __init__(self,
                 num_classes,
                 pre_process=True,
                 post_process=True):
        super(SegformerSegmentationModel, self).__init__()
        args = get_args()
        self.hidden_size = args.hidden_size
        self.num_classes = num_classes
        self.pre_process = pre_process
        self.post_process = post_process

        self.backbone = mit_b5()
        self.head = SegformerSegmentationHead(
            feature_strides=[4, 8, 16, 32],
            in_channels=[64, 128, 320, 512],
            embedding_dim=768,
            dropout_ratio=0.1
        )

    def set_input_tensor(self, input_tensor):
        """See megatron.model.transformer.set_input_tensor()"""
        pass

    def forward(self, input):
        # [b hw c]
        hidden_states = self.backbone(input)
        hidden_states = self.head(hidden_states)
        return hidden_states


    def __init__(self,
                 num_classes,
                 pre_process=True,
                 post_process=True):
        super(SegformerSegmentationModel, self).__init__()
        args = get_args()
        self.hidden_size = args.hidden_size
        self.num_classes = num_classes
        self.pre_process = pre_process
        self.post_process = post_process

        self.backbone = mit_b5()
        self.head = SegformerSegmentationHead(
            feature_strides=[4, 8, 16, 32],
            in_channels=[64, 128, 320, 512],
            embedding_dim=768,
            dropout_ratio=0.1
        )

    def set_input_tensor(self, input_tensor):
        """See megatron.model.transformer.set_input_tensor()"""
        pass

    def forward(self, input):
        # [b hw c]
        hidden_states = self.backbone(input)
        hidden_states = self.head(hidden_states)
        return hidden_states


forward

                 post_process=True):
        super(SegformerSegmentationModel, self).__init__()
        args = get_args()
        self.hidden_size = args.hidden_size
        self.num_classes = num_classes
        self.pre_process = pre_process
        self.post_process = post_process

        self.backbone = mit_b5()
        self.head = SegformerSegmentationHead(
            feature_strides=[4, 8, 16, 32],
            in_channels=[64, 128, 320, 512],
            embedding_dim=768,
            dropout_ratio=0.1
        )

    def set_input_tensor(self, input_tensor):
        """See megatron.model.transformer.set_input_tensor()"""
        pass

    def forward(self, input):
        # [b hw c]
        hidden_states = self.backbone(input)
        hidden_states = self.head(hidden_states)
        return hidden_states


tasks.vision.segmentation.utils

import math
import torch
import numpy as np
from megatron import get_args

def slidingcrops(img, mask):
    # img: [b c h w]
    # mask: [b h w]
    args = get_args()
    assert args.img_h == args.img_w
    crop_size = args.img_h
    stride = args.seg_stride
    ignore_index = args.ignore_index
    n, c, h, w = img.shape
    assert h >= crop_size
    assert w >= crop_size
    long_size = max(h, w)

    img_slices, mask_slices, slices_info = [], [], []
    if long_size > crop_size:
        assert stride <= crop_size
        h_step_num = int(math.ceil((h - crop_size) / float(stride))) + 1
        w_step_num = int(math.ceil((w - crop_size) / float(stride))) + 1
        for yy in range(h_step_num):
            for xx in range(w_step_num):
                sy, sx = yy * stride, xx * stride
                ey, ex = sy + crop_size, sx + crop_size
                img_sub = img[:, :, sy: ey, sx: ex]
                mask_sub = mask[:, sy: ey, sx: ex]

                # padding
                sub_h, sub_w = img_sub.shape[2:]
                pad_h = max(crop_size - sub_h, 0)
                pad_w = max(crop_size - sub_w, 0)
                img_sub = torch.nn.functional.pad(img_sub, pad=(0, pad_w, 0, pad_h), value=ignore_index)
                mask_sub = torch.nn.functional.pad(mask_sub, pad=(0, pad_w, 0, pad_h))

                img_slices.append(img_sub)
                mask_slices.append(mask_sub)
                slices_info.append([sy, ey, sx, ex, sub_h, sub_w])

        return torch.cat(img_slices), torch.cat(mask_slices), slices_info, (h, w)
    else:
        return img, mask, [[0, h, 0, w, h, w]], (h, w)

def slidingjoins(preds, probs, labels, slices_info, img_size):
    args = get_args()
    num_slices = len(slices_info)

    if num_slices == 1:
        return preds, labels

    h, w = img_size
    split_size = args.micro_batch_size

    preds_split = torch.split(preds, split_size)
    probs_split = torch.split(probs, split_size)
    labels_split = torch.split(labels, split_size)

    assert(len(preds_split) == num_slices)

    total_max_probs = torch.zeros((split_size, h, w), dtype=torch.float, device='cuda')
    total_preds = torch.zeros((split_size, h, w), dtype=torch.int, device='cuda')
    total_labels = torch.zeros((split_size, h, w), dtype=torch.int, device='cuda')

    for i in range(num_slices):
        sy, ey, sx, ex, sub_h, sub_w = slices_info[i]
        assert sy + sub_h <= h
        assert sx + sub_w <= w
        curr_max_probs = total_max_probs[:, sy:sy + sub_h, sx:sx + sub_w]
        curr_preds = total_preds[:, sy:sy + sub_h, sx:sx + sub_w]

        local_max_probs = probs_split[i][:, :sub_h, : sub_w]
        local_preds = preds_split[i][:, :sub_h, :sub_w]

        result_max_probs = torch.maximum(curr_max_probs, local_max_probs)
        result_preds = torch.where(curr_max_probs >= local_max_probs, curr_preds, local_preds)

        total_max_probs[:, sy:sy + sub_h, sx:sx + sub_w] = result_max_probs
        total_preds[:, sy:sy + sub_h, sx:sx + sub_w] = result_preds
        total_labels[:, sy:sy + sub_h, sx:sx + sub_w] = labels_split[i][0, :sub_h, :sub_w]

    return total_preds, total_labels


slidingcrops

import math
import torch
import numpy as np
from megatron import get_args

def slidingcrops(img, mask):
    # img: [b c h w]
    # mask: [b h w]
    args = get_args()
    assert args.img_h == args.img_w
    crop_size = args.img_h
    stride = args.seg_stride
    ignore_index = args.ignore_index
    n, c, h, w = img.shape
    assert h >= crop_size
    assert w >= crop_size
    long_size = max(h, w)

    img_slices, mask_slices, slices_info = [], [], []
    if long_size > crop_size:
        assert stride <= crop_size
        h_step_num = int(math.ceil((h - crop_size) / float(stride))) + 1
        w_step_num = int(math.ceil((w - crop_size) / float(stride))) + 1
        for yy in range(h_step_num):
            for xx in range(w_step_num):
                sy, sx = yy * stride, xx * stride
                ey, ex = sy + crop_size, sx + crop_size
                img_sub = img[:, :, sy: ey, sx: ex]
                mask_sub = mask[:, sy: ey, sx: ex]

                # padding
                sub_h, sub_w = img_sub.shape[2:]
                pad_h = max(crop_size - sub_h, 0)
                pad_w = max(crop_size - sub_w, 0)
                img_sub = torch.nn.functional.pad(img_sub, pad=(0, pad_w, 0, pad_h), value=ignore_index)
                mask_sub = torch.nn.functional.pad(mask_sub, pad=(0, pad_w, 0, pad_h))

                img_slices.append(img_sub)
                mask_slices.append(mask_sub)
                slices_info.append([sy, ey, sx, ex, sub_h, sub_w])

        return torch.cat(img_slices), torch.cat(mask_slices), slices_info, (h, w)
    else:
        return img, mask, [[0, h, 0, w, h, w]], (h, w)

def slidingjoins(preds, probs, labels, slices_info, img_size):
    args = get_args()
    num_slices = len(slices_info)

    if num_slices == 1:
        return preds, labels

    h, w = img_size
    split_size = args.micro_batch_size

    preds_split = torch.split(preds, split_size)
    probs_split = torch.split(probs, split_size)
    labels_split = torch.split(labels, split_size)

    assert(len(preds_split) == num_slices)

    total_max_probs = torch.zeros((split_size, h, w), dtype=torch.float, device='cuda')
    total_preds = torch.zeros((split_size, h, w), dtype=torch.int, device='cuda')


slidingjoins

                ey, ex = sy + crop_size, sx + crop_size
                img_sub = img[:, :, sy: ey, sx: ex]
                mask_sub = mask[:, sy: ey, sx: ex]

                # padding
                sub_h, sub_w = img_sub.shape[2:]
                pad_h = max(crop_size - sub_h, 0)
                pad_w = max(crop_size - sub_w, 0)
                img_sub = torch.nn.functional.pad(img_sub, pad=(0, pad_w, 0, pad_h), value=ignore_index)
                mask_sub = torch.nn.functional.pad(mask_sub, pad=(0, pad_w, 0, pad_h))

                img_slices.append(img_sub)
                mask_slices.append(mask_sub)
                slices_info.append([sy, ey, sx, ex, sub_h, sub_w])

        return torch.cat(img_slices), torch.cat(mask_slices), slices_info, (h, w)
    else:
        return img, mask, [[0, h, 0, w, h, w]], (h, w)

def slidingjoins(preds, probs, labels, slices_info, img_size):
    args = get_args()
    num_slices = len(slices_info)

    if num_slices == 1:
        return preds, labels

    h, w = img_size
    split_size = args.micro_batch_size

    preds_split = torch.split(preds, split_size)
    probs_split = torch.split(probs, split_size)
    labels_split = torch.split(labels, split_size)

    assert(len(preds_split) == num_slices)

    total_max_probs = torch.zeros((split_size, h, w), dtype=torch.float, device='cuda')
    total_preds = torch.zeros((split_size, h, w), dtype=torch.int, device='cuda')
    total_labels = torch.zeros((split_size, h, w), dtype=torch.int, device='cuda')

    for i in range(num_slices):
        sy, ey, sx, ex, sub_h, sub_w = slices_info[i]
        assert sy + sub_h <= h
        assert sx + sub_w <= w
        curr_max_probs = total_max_probs[:, sy:sy + sub_h, sx:sx + sub_w]
        curr_preds = total_preds[:, sy:sy + sub_h, sx:sx + sub_w]

        local_max_probs = probs_split[i][:, :sub_h, : sub_w]
        local_preds = preds_split[i][:, :sub_h, :sub_w]

        result_max_probs = torch.maximum(curr_max_probs, local_max_probs)
        result_preds = torch.where(curr_max_probs >= local_max_probs, curr_preds, local_preds)

        total_max_probs[:, sy:sy + sub_h, sx:sx + sub_w] = result_max_probs
        total_preds[:, sy:sy + sub_h, sx:sx + sub_w] = result_preds
        total_labels[:, sy:sy + sub_h, sx:sx + sub_w] = labels_split[i][0, :sub_h, :sub_w]

    return total_preds, total_labels


tasks.vision.segmentation.data

import random
import os
import math
import mmcv
import torch
import numpy as np
import torchvision.transforms as T
from torchvision import datasets
from torch.utils.data import Dataset
from megatron.data.autoaugment import ImageNetPolicy
from tasks.vision.segmentation.cityscapes import Cityscapes
import tasks.vision.segmentation.transforms as ET
from megatron.data.autoaugment import ImageNetPolicy
from megatron import get_args
from PIL import Image, ImageOps

class VitSegmentationJointTransform():
    def __init__(self, train=True, resolution=None):
        self.train = train
        if self.train:
            self.transform0 = ET.RandomSizeAndCrop(resolution)
            self.transform1 = ET.RandomHorizontallyFlip()

    def __call__(self, img, mask):
        if self.train:
            img, mask = self.transform0(img, mask)
            img, mask = self.transform1(img, mask)
        return img, mask

class VitSegmentationImageTransform():
    def __init__(self, train=True, resolution=None):
        args = get_args()
        self.train = train
        assert args.fp16 or args.bf16
        self.data_type = torch.half if args.fp16 else torch.bfloat16
        self.mean_std = args.mean_std
        if self.train:
            assert resolution is not None
            self.transform = T.Compose([
                ET.PhotoMetricDistortion(),
                T.ToTensor(),
                T.Normalize(*self.mean_std),
                T.ConvertImageDtype(self.data_type)
            ])
        else:
            self.transform = T.Compose([
                T.ToTensor(),
                T.Normalize(*self.mean_std),
                T.ConvertImageDtype(self.data_type)
            ])

    def __call__(self, input):
        output = self.transform(input)
        return output

class VitSegmentationTargetTransform():
    def __init__(self, train=True, resolution=None):
        self.train = train

    def __call__(self, input):
        output = torch.from_numpy(np.array(input, dtype=np.int32)).long()
        return output

class RandomSeedSegmentationDataset(Dataset):
    def __init__(self,
                 dataset,
                 joint_transform,
                 image_transform,
                 target_transform):

        args = get_args()
        self.base_seed = args.seed
        self.curr_seed = self.base_seed
        self.dataset = dataset
        self.joint_transform = joint_transform
        self.image_transform = image_transform
        self.target_transform = target_transform

    def __len__(self):
        return len(self.dataset)

    def set_epoch(self, epoch):
        self.curr_seed = self.base_seed + 100 * epoch

    def __getitem__(self, idx):
        seed = idx + self.curr_seed
        img, mask = self.dataset[idx]

        torch.manual_seed(seed)
        random.seed(seed)
        np.random.seed(seed)
        img, mask = self.joint_transform(img, mask)
        img = self.image_transform(img)
        mask = self.target_transform(mask)

        return img, mask

def build_cityscapes_train_valid_datasets(data_path, image_size):
    args = get_args()
    args.num_classes = Cityscapes.num_classes
    args.ignore_index = Cityscapes.ignore_index
    args.color_table = Cityscapes.color_table
    args.mean_std = ([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])

    train_joint_transform = \
        VitSegmentationJointTransform(train=True, resolution=image_size)
    val_joint_transform = \
        VitSegmentationJointTransform(train=False, resolution=image_size)
    train_image_transform = \
        VitSegmentationImageTransform(train=True, resolution=image_size)
    val_image_transform = \
        VitSegmentationImageTransform(train=False, resolution=image_size)
    train_target_transform = \
        VitSegmentationTargetTransform(train=True, resolution=image_size)
    val_target_transform = \
        VitSegmentationTargetTransform(train=False, resolution=image_size)

    # training dataset
    train_data = Cityscapes(
        root=data_path[0],
        split='train',
        mode='fine',
        resolution=image_size
    )
    train_data = RandomSeedSegmentationDataset(
        train_data,
        joint_transform=train_joint_transform,
        image_transform=train_image_transform,
        target_transform=train_target_transform)

    # validation dataset
    val_data = Cityscapes(
        root=data_path[0],
        split='val',
        mode='fine',
        resolution=image_size
    )

    val_data = RandomSeedSegmentationDataset(
        val_data,
        joint_transform=val_joint_transform,
        image_transform=val_image_transform,
        target_transform=val_target_transform)

    return train_data, val_data

def build_train_valid_datasets(data_path, image_size):
    return build_cityscapes_train_valid_datasets(data_path, image_size)


VitSegmentationJointTransform

import random
import os
import math
import mmcv
import torch
import numpy as np
import torchvision.transforms as T
from torchvision import datasets
from torch.utils.data import Dataset
from megatron.data.autoaugment import ImageNetPolicy
from tasks.vision.segmentation.cityscapes import Cityscapes
import tasks.vision.segmentation.transforms as ET
from megatron.data.autoaugment import ImageNetPolicy
from megatron import get_args
from PIL import Image, ImageOps

class VitSegmentationJointTransform():
    def __init__(self, train=True, resolution=None):
        self.train = train
        if self.train:
            self.transform0 = ET.RandomSizeAndCrop(resolution)
            self.transform1 = ET.RandomHorizontallyFlip()

    def __call__(self, img, mask):
        if self.train:
            img, mask = self.transform0(img, mask)
            img, mask = self.transform1(img, mask)
        return img, mask

class VitSegmentationImageTransform():
    def __init__(self, train=True, resolution=None):
        args = get_args()
        self.train = train
        assert args.fp16 or args.bf16
        self.data_type = torch.half if args.fp16 else torch.bfloat16
        self.mean_std = args.mean_std
        if self.train:
            assert resolution is not None
            self.transform = T.Compose([
                ET.PhotoMetricDistortion(),
                T.ToTensor(),
                T.Normalize(*self.mean_std),
                T.ConvertImageDtype(self.data_type)
            ])
        else:
            self.transform = T.Compose([
                T.ToTensor(),


VitSegmentationImageTransform

import tasks.vision.segmentation.transforms as ET
from megatron.data.autoaugment import ImageNetPolicy
from megatron import get_args
from PIL import Image, ImageOps

class VitSegmentationJointTransform():
    def __init__(self, train=True, resolution=None):
        self.train = train
        if self.train:
            self.transform0 = ET.RandomSizeAndCrop(resolution)
            self.transform1 = ET.RandomHorizontallyFlip()

    def __call__(self, img, mask):
        if self.train:
            img, mask = self.transform0(img, mask)
            img, mask = self.transform1(img, mask)
        return img, mask

class VitSegmentationImageTransform():
    def __init__(self, train=True, resolution=None):
        args = get_args()
        self.train = train
        assert args.fp16 or args.bf16
        self.data_type = torch.half if args.fp16 else torch.bfloat16
        self.mean_std = args.mean_std
        if self.train:
            assert resolution is not None
            self.transform = T.Compose([
                ET.PhotoMetricDistortion(),
                T.ToTensor(),
                T.Normalize(*self.mean_std),
                T.ConvertImageDtype(self.data_type)
            ])
        else:
            self.transform = T.Compose([
                T.ToTensor(),
                T.Normalize(*self.mean_std),
                T.ConvertImageDtype(self.data_type)
            ])

    def __call__(self, input):
        output = self.transform(input)
        return output

class VitSegmentationTargetTransform():
    def __init__(self, train=True, resolution=None):
        self.train = train

    def __call__(self, input):
        output = torch.from_numpy(np.array(input, dtype=np.int32)).long()
        return output

class RandomSeedSegmentationDataset(Dataset):
    def __init__(self,
                 dataset,
                 joint_transform,
                 image_transform,
                 target_transform):

        args = get_args()
        self.base_seed = args.seed


VitSegmentationTargetTransform

        if self.train:
            assert resolution is not None
            self.transform = T.Compose([
                ET.PhotoMetricDistortion(),
                T.ToTensor(),
                T.Normalize(*self.mean_std),
                T.ConvertImageDtype(self.data_type)
            ])
        else:
            self.transform = T.Compose([
                T.ToTensor(),
                T.Normalize(*self.mean_std),
                T.ConvertImageDtype(self.data_type)
            ])

    def __call__(self, input):
        output = self.transform(input)
        return output

class VitSegmentationTargetTransform():
    def __init__(self, train=True, resolution=None):
        self.train = train

    def __call__(self, input):
        output = torch.from_numpy(np.array(input, dtype=np.int32)).long()
        return output

class RandomSeedSegmentationDataset(Dataset):
    def __init__(self,
                 dataset,
                 joint_transform,
                 image_transform,
                 target_transform):

        args = get_args()
        self.base_seed = args.seed
        self.curr_seed = self.base_seed
        self.dataset = dataset
        self.joint_transform = joint_transform
        self.image_transform = image_transform
        self.target_transform = target_transform

    def __len__(self):
        return len(self.dataset)


RandomSeedSegmentationDataset

            self.transform = T.Compose([
                T.ToTensor(),
                T.Normalize(*self.mean_std),
                T.ConvertImageDtype(self.data_type)
            ])

    def __call__(self, input):
        output = self.transform(input)
        return output

class VitSegmentationTargetTransform():
    def __init__(self, train=True, resolution=None):
        self.train = train

    def __call__(self, input):
        output = torch.from_numpy(np.array(input, dtype=np.int32)).long()
        return output

class RandomSeedSegmentationDataset(Dataset):
    def __init__(self,
                 dataset,
                 joint_transform,
                 image_transform,
                 target_transform):

        args = get_args()
        self.base_seed = args.seed
        self.curr_seed = self.base_seed
        self.dataset = dataset
        self.joint_transform = joint_transform
        self.image_transform = image_transform
        self.target_transform = target_transform

    def __len__(self):
        return len(self.dataset)

    def set_epoch(self, epoch):
        self.curr_seed = self.base_seed + 100 * epoch

    def __getitem__(self, idx):
        seed = idx + self.curr_seed
        img, mask = self.dataset[idx]

        torch.manual_seed(seed)
        random.seed(seed)
        np.random.seed(seed)
        img, mask = self.joint_transform(img, mask)
        img = self.image_transform(img)
        mask = self.target_transform(mask)

        return img, mask

def build_cityscapes_train_valid_datasets(data_path, image_size):
    args = get_args()
    args.num_classes = Cityscapes.num_classes
    args.ignore_index = Cityscapes.ignore_index
    args.color_table = Cityscapes.color_table
    args.mean_std = ([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])

    train_joint_transform = \
        VitSegmentationJointTransform(train=True, resolution=image_size)
    val_joint_transform = \
        VitSegmentationJointTransform(train=False, resolution=image_size)
    train_image_transform = \
        VitSegmentationImageTransform(train=True, resolution=image_size)
    val_image_transform = \
        VitSegmentationImageTransform(train=False, resolution=image_size)
    train_target_transform = \
        VitSegmentationTargetTransform(train=True, resolution=image_size)
    val_target_transform = \


    def __len__(self):
        return len(self.dataset)

    def set_epoch(self, epoch):
        self.curr_seed = self.base_seed + 100 * epoch

    def __getitem__(self, idx):
        seed = idx + self.curr_seed
        img, mask = self.dataset[idx]

        torch.manual_seed(seed)
        random.seed(seed)
        np.random.seed(seed)
        img, mask = self.joint_transform(img, mask)
        img = self.image_transform(img)
        mask = self.target_transform(mask)

        return img, mask

def build_cityscapes_train_valid_datasets(data_path, image_size):
    args = get_args()
    args.num_classes = Cityscapes.num_classes
    args.ignore_index = Cityscapes.ignore_index
    args.color_table = Cityscapes.color_table
    args.mean_std = ([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])

    train_joint_transform = \
        VitSegmentationJointTransform(train=True, resolution=image_size)
    val_joint_transform = \
        VitSegmentationJointTransform(train=False, resolution=image_size)
    train_image_transform = \
        VitSegmentationImageTransform(train=True, resolution=image_size)
    val_image_transform = \
        VitSegmentationImageTransform(train=False, resolution=image_size)
    train_target_transform = \
        VitSegmentationTargetTransform(train=True, resolution=image_size)
    val_target_transform = \
        VitSegmentationTargetTransform(train=False, resolution=image_size)

    # training dataset
    train_data = Cityscapes(
        root=data_path[0],
        split='train',
        mode='fine',
        resolution=image_size
    )
    train_data = RandomSeedSegmentationDataset(
        train_data,
        joint_transform=train_joint_transform,
        image_transform=train_image_transform,
        target_transform=train_target_transform)

    # validation dataset
    val_data = Cityscapes(
        root=data_path[0],
        split='val',
        mode='fine',
        resolution=image_size
    )

    val_data = RandomSeedSegmentationDataset(
        val_data,
        joint_transform=val_joint_transform,
        image_transform=val_image_transform,
        target_transform=val_target_transform)

    return train_data, val_data

def build_train_valid_datasets(data_path, image_size):
    return build_cityscapes_train_valid_datasets(data_path, image_size)


        image_transform=train_image_transform,
        target_transform=train_target_transform)

    # validation dataset
    val_data = Cityscapes(
        root=data_path[0],
        split='val',
        mode='fine',
        resolution=image_size
    )

    val_data = RandomSeedSegmentationDataset(
        val_data,
        joint_transform=val_joint_transform,
        image_transform=val_image_transform,
        target_transform=val_target_transform)

    return train_data, val_data

def build_train_valid_datasets(data_path, image_size):
    return build_cityscapes_train_valid_datasets(data_path, image_size)


                T.ToTensor(),
                T.Normalize(*self.mean_std),
                T.ConvertImageDtype(self.data_type)
            ])

    def __call__(self, input):
        output = self.transform(input)
        return output

class VitSegmentationTargetTransform():
    def __init__(self, train=True, resolution=None):
        self.train = train

    def __call__(self, input):
        output = torch.from_numpy(np.array(input, dtype=np.int32)).long()
        return output

class RandomSeedSegmentationDataset(Dataset):
    def __init__(self,
                 dataset,
                 joint_transform,
                 image_transform,
                 target_transform):

        args = get_args()
        self.base_seed = args.seed
        self.curr_seed = self.base_seed
        self.dataset = dataset
        self.joint_transform = joint_transform
        self.image_transform = image_transform
        self.target_transform = target_transform

    def __len__(self):
        return len(self.dataset)

    def set_epoch(self, epoch):
        self.curr_seed = self.base_seed + 100 * epoch

    def __getitem__(self, idx):
        seed = idx + self.curr_seed
        img, mask = self.dataset[idx]

        torch.manual_seed(seed)
        random.seed(seed)
        np.random.seed(seed)
        img, mask = self.joint_transform(img, mask)
        img = self.image_transform(img)
        mask = self.target_transform(mask)

        return img, mask


                T.ToTensor(),
                T.Normalize(*self.mean_std),
                T.ConvertImageDtype(self.data_type)
            ])
        else:
            self.transform = T.Compose([
                T.ToTensor(),
                T.Normalize(*self.mean_std),
                T.ConvertImageDtype(self.data_type)
            ])

    def __call__(self, input):
        output = self.transform(input)
        return output

class VitSegmentationTargetTransform():
    def __init__(self, train=True, resolution=None):
        self.train = train

    def __call__(self, input):
        output = torch.from_numpy(np.array(input, dtype=np.int32)).long()
        return output

class RandomSeedSegmentationDataset(Dataset):
    def __init__(self,
                 dataset,
                 joint_transform,
                 image_transform,
                 target_transform):

        args = get_args()
        self.base_seed = args.seed
        self.curr_seed = self.base_seed
        self.dataset = dataset
        self.joint_transform = joint_transform
        self.image_transform = image_transform
        self.target_transform = target_transform

    def __len__(self):
        return len(self.dataset)


    def __call__(self, input):
        output = torch.from_numpy(np.array(input, dtype=np.int32)).long()
        return output

class RandomSeedSegmentationDataset(Dataset):
    def __init__(self,
                 dataset,
                 joint_transform,
                 image_transform,
                 target_transform):

        args = get_args()
        self.base_seed = args.seed
        self.curr_seed = self.base_seed
        self.dataset = dataset
        self.joint_transform = joint_transform
        self.image_transform = image_transform
        self.target_transform = target_transform

    def __len__(self):
        return len(self.dataset)

    def set_epoch(self, epoch):
        self.curr_seed = self.base_seed + 100 * epoch

    def __getitem__(self, idx):
        seed = idx + self.curr_seed
        img, mask = self.dataset[idx]

        torch.manual_seed(seed)
        random.seed(seed)
        np.random.seed(seed)
        img, mask = self.joint_transform(img, mask)
        img = self.image_transform(img)
        mask = self.target_transform(mask)

        return img, mask

def build_cityscapes_train_valid_datasets(data_path, image_size):
    args = get_args()


class RandomSeedSegmentationDataset(Dataset):
    def __init__(self,
                 dataset,
                 joint_transform,
                 image_transform,
                 target_transform):

        args = get_args()
        self.base_seed = args.seed
        self.curr_seed = self.base_seed
        self.dataset = dataset
        self.joint_transform = joint_transform
        self.image_transform = image_transform
        self.target_transform = target_transform

    def __len__(self):
        return len(self.dataset)

    def set_epoch(self, epoch):
        self.curr_seed = self.base_seed + 100 * epoch

    def __getitem__(self, idx):
        seed = idx + self.curr_seed
        img, mask = self.dataset[idx]

        torch.manual_seed(seed)
        random.seed(seed)
        np.random.seed(seed)
        img, mask = self.joint_transform(img, mask)
        img = self.image_transform(img)
        mask = self.target_transform(mask)

        return img, mask

def build_cityscapes_train_valid_datasets(data_path, image_size):
    args = get_args()
    args.num_classes = Cityscapes.num_classes
    args.ignore_index = Cityscapes.ignore_index
    args.color_table = Cityscapes.color_table


    def __init__(self,
                 dataset,
                 joint_transform,
                 image_transform,
                 target_transform):

        args = get_args()
        self.base_seed = args.seed
        self.curr_seed = self.base_seed
        self.dataset = dataset
        self.joint_transform = joint_transform
        self.image_transform = image_transform
        self.target_transform = target_transform

    def __len__(self):
        return len(self.dataset)

    def set_epoch(self, epoch):
        self.curr_seed = self.base_seed + 100 * epoch

    def __getitem__(self, idx):
        seed = idx + self.curr_seed
        img, mask = self.dataset[idx]

        torch.manual_seed(seed)
        random.seed(seed)
        np.random.seed(seed)
        img, mask = self.joint_transform(img, mask)
        img = self.image_transform(img)
        mask = self.target_transform(mask)

        return img, mask

def build_cityscapes_train_valid_datasets(data_path, image_size):
    args = get_args()
    args.num_classes = Cityscapes.num_classes
    args.ignore_index = Cityscapes.ignore_index
    args.color_table = Cityscapes.color_table
    args.mean_std = ([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])

    train_joint_transform = \
        VitSegmentationJointTransform(train=True, resolution=image_size)
    val_joint_transform = \
        VitSegmentationJointTransform(train=False, resolution=image_size)
    train_image_transform = \
        VitSegmentationImageTransform(train=True, resolution=image_size)
    val_image_transform = \
        VitSegmentationImageTransform(train=False, resolution=image_size)
    train_target_transform = \
        VitSegmentationTargetTransform(train=True, resolution=image_size)
    val_target_transform = \


tasks.vision.segmentation.metrics

#!/usr/bin/env python
# -*- coding: UTF-8 -*-
#copyright (c) go-hiroaki & Chokurei
#email: guangmingwu2010@gmail.com
#       guozhilingty@gmail.com
#
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
import math
import torch
import torch.nn as nn
import torch.nn.functional as F

eps = 1e-6

def _binarize(y_data, threshold):
    """
    args:
        y_data : [float] 4-d tensor in [batch_size, channels, img_rows, img_cols]
        threshold : [float] [0.0, 1.0]
    return 4-d binarized y_data
    """
    y_data[y_data < threshold] = 0.0
    y_data[y_data >= threshold] = 1.0
    return y_data

def _argmax(y_data, dim):
    """
    args:
        y_data : 4-d tensor in [batch_size, chs, img_rows, img_cols]
        dim : int
    return 3-d [int] y_data
    """
    return torch.argmax(y_data, dim).int()

def _get_tp(y_pred, y_true):
    """
    args:
        y_true : [int] 3-d in [batch_size, img_rows, img_cols]
        y_pred : [int] 3-d in [batch_size, img_rows, img_cols]
    return [float] true_positive
    """
    return torch.sum(y_true * y_pred).float()

def _get_fp(y_pred, y_true):
    """
    args:
        y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
        y_pred : 3-d ndarray in [batch_size, img_rows, img_cols]
    return [float] false_positive
    """
    return torch.sum((1 - y_true) * y_pred).float()

def _get_tn(y_pred, y_true):
    """
    args:
        y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
        y_pred : 3-d ndarray in [batch_size, img_rows, img_cols]
    return [float] true_negative
    """
    return torch.sum((1 - y_true) * (1 - y_pred)).float()

def _get_fn(y_pred, y_true):
    """
    args:
        y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
        y_pred : 3-d ndarray in [batch_size, img_rows, img_cols]
    return [float] false_negative
    """
    return torch.sum(y_true * (1 - y_pred)).float()

def _get_weights(y_true, nb_ch):
    """
    args:
        y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
        nb_ch : int
    return [float] weights
    """
    batch_size, img_rows, img_cols = y_true.shape
    pixels = batch_size * img_rows * img_cols
    weights = [torch.sum(y_true==ch).item() / pixels for ch in range(nb_ch)]
    return weights

class CFMatrix(object):
    def __init__(self, des=None):
        self.des = des

    def __repr__(self):
        return "ConfusionMatrix"

    def __call__(self, y_pred, y_true, ignore_index, threshold=0.5):

        """
        args:
            y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
            y_pred : 4-d ndarray in [batch_size, chs, img_rows, img_cols]
            threshold : [0.0, 1.0]
        return confusion matrix
        """
        batch_size, img_rows, img_cols = y_pred.shape
        chs = ignore_index
        device = y_true.device
        if chs == 1:
            y_pred = _binarize(y_pred, threshold)
            y_true = _binarize(y_true, threshold)
            nb_tp = _get_tp(y_pred, y_true)
            nb_fp = _get_fp(y_pred, y_true)
            nb_tn = _get_tn(y_pred, y_true)
            nb_fn = _get_fn(y_pred, y_true)
            mperforms = [nb_tp, nb_fp, nb_tn, nb_fn]
            performs = None
        else:
            performs = torch.zeros(chs, 4).to(device)
            weights = _get_weights(y_true, chs)
            for ch in range(chs):
                y_true_ch = torch.zeros(batch_size, img_rows, img_cols)
                y_false_ch = torch.zeros(batch_size, img_rows, img_cols)
                y_pred_ch = torch.zeros(batch_size, img_rows, img_cols)
                y_true_ch[y_true == ch] = 1
                y_false_ch[torch.logical_and((y_true != ch), (y_true != ignore_index))] = 1
                y_pred_ch[y_pred == ch] = 1
                nb_tp = _get_tp(y_pred_ch, y_true_ch)
                nb_fp = torch.sum(y_false_ch * y_pred_ch).float()
                nb_tn = torch.sum(y_false_ch * (1 - y_pred_ch)).float()
                nb_fn = _get_fn(y_pred_ch, y_true_ch)
                performs[int(ch), :] = torch.FloatTensor([nb_tp, nb_fp, nb_tn, nb_fn])
            mperforms = sum([i*j for (i, j) in zip(performs, weights)])
        return mperforms, performs

class OAAcc(object):
    def __init__(self, des="Overall Accuracy"):
        self.des = des

    def __repr__(self):
        return "OAcc"

    def __call__(self, y_pred, y_true, threshold=0.5):
        """
        args:
            y_true : 4-d ndarray in [batch_size, chs, img_rows, img_cols]
            y_pred : 4-d ndarray in [batch_size, chs, img_rows, img_cols]
            threshold : [0.0, 1.0]
        return (tp+tn)/total
        """
        batch_size, chs, img_rows, img_cols = y_true.shape
        device = y_true.device
        if chs == 1:
            y_pred = _binarize(y_pred, threshold)
            y_true = _binarize(y_true, threshold)
        else:
            y_pred = _argmax(y_pred, 1)
            y_true = _argmax(y_true, 1)

        nb_tp_tn = torch.sum(y_true == y_pred).float()
        mperforms = nb_tp_tn / (batch_size * img_rows * img_cols)
        performs = None
        return mperforms, performs

class Precision(object):
    def __init__(self, des="Precision"):
        self.des = des

    def __repr__(self):
        return "Prec"

    def __call__(self, y_pred, y_true, threshold=0.5):
        """
        args:
            y_true : 4-d ndarray in [batch_size, chs, img_rows, img_cols]
            y_pred : 4-d ndarray in [batch_size, chs, img_rows, img_cols]
            threshold : [0.0, 1.0]
        return tp/(tp+fp)
        """
        batch_size, chs, img_rows, img_cols = y_true.shape
        device = y_true.device
        if chs == 1:
            y_pred = _binarize(y_pred, threshold)
            y_true = _binarize(y_true, threshold)
            nb_tp = _get_tp(y_pred, y_true)
            nb_fp = _get_fp(y_pred, y_true)
            mperforms = nb_tp / (nb_tp + nb_fp + esp)
            performs = None
        else:
            y_pred = _argmax(y_pred, 1)
            y_true = _argmax(y_true, 1)
            performs = torch.zeros(chs, 1).to(device)
            weights = _get_weights(y_true, chs)
            for ch in range(chs):
                y_true_ch = torch.zeros(batch_size, img_rows, img_cols)
                y_pred_ch = torch.zeros(batch_size, img_rows, img_cols)
                y_true_ch[y_true == ch] = 1
                y_pred_ch[y_pred == ch] = 1
                nb_tp = _get_tp(y_pred_ch, y_true_ch)
                nb_fp = _get_fp(y_pred_ch, y_true_ch)
                performs[int(ch)] = nb_tp / (nb_tp + nb_fp + esp)
            mperforms = sum([i*j for (i, j) in zip(performs, weights)])
        return mperforms, performs

class Recall(object):
    def __init__(self, des="Recall"):
        self.des = des

    def __repr__(self):
        return "Reca"

    def __call__(self, y_pred, y_true, threshold=0.5):
        """
        args:
            y_true : 4-d ndarray in [batch_size, chs, img_rows, img_cols]
            y_pred : 4-d ndarray in [batch_size, chs, img_rows, img_cols]
            threshold : [0.0, 1.0]
        return tp/(tp+fn)
        """
        batch_size, chs, img_rows, img_cols = y_true.shape
        device = y_true.device
        if chs == 1:
            y_pred = _binarize(y_pred, threshold)
            y_true = _binarize(y_true, threshold)
            nb_tp = _get_tp(y_pred, y_true)
            nb_fn = _get_fn(y_pred, y_true)
            mperforms = nb_tp / (nb_tp + nb_fn + esp)
            performs = None
        else:
            y_pred = _argmax(y_pred, 1)
            y_true = _argmax(y_true, 1)
            performs = torch.zeros(chs, 1).to(device)
            weights = _get_weights(y_true, chs)
            for ch in range(chs):
                y_true_ch = torch.zeros(batch_size, img_rows, img_cols)
                y_pred_ch = torch.zeros(batch_size, img_rows, img_cols)
                y_true_ch[y_true == ch] = 1
                y_pred_ch[y_pred == ch] = 1
                nb_tp = _get_tp(y_pred_ch, y_true_ch)
                nb_fn = _get_fn(y_pred_ch, y_true_ch)
                performs[int(ch)] = nb_tp / (nb_tp + nb_fn + esp)
            mperforms = sum([i*j for (i, j) in zip(performs, weights)])
        return mperforms, performs

class F1Score(object):
    def __init__(self, des="F1Score"):
        self.des = des

    def __repr__(self):
        return "F1Sc"

    def __call__(self, y_pred, y_true, threshold=0.5):

        """
        args:
            y_true : 4-d ndarray in [batch_size, chs, img_rows, img_cols]
            y_pred : 4-d ndarray in [batch_size, chs, img_rows, img_cols]
            threshold : [0.0, 1.0]
        return 2*precision*recall/(precision+recall)
        """
        batch_size, chs, img_rows, img_cols = y_true.shape
        device = y_true.device
        if chs == 1:
            y_pred = _binarize(y_pred, threshold)
            y_true = _binarize(y_true, threshold)
            nb_tp = _get_tp(y_pred, y_true)
            nb_fp = _get_fp(y_pred, y_true)
            nb_fn = _get_fn(y_pred, y_true)
            _precision = nb_tp / (nb_tp + nb_fp + esp)
            _recall = nb_tp / (nb_tp + nb_fn + esp)
            mperforms = 2 * _precision * _recall / (_precision + _recall + esp)
            performs = None
        else:
            y_pred = _argmax(y_pred, 1)
            y_true = _argmax(y_true, 1)
            performs = torch.zeros(chs, 1).to(device)
            weights = _get_weights(y_true, chs)
            for ch in range(chs):
                y_true_ch = torch.zeros(batch_size, img_rows, img_cols)
                y_pred_ch = torch.zeros(batch_size, img_rows, img_cols)
                y_true_ch[y_true == ch] = 1
                y_pred_ch[y_pred == ch] = 1
                nb_tp = _get_tp(y_pred_ch, y_true_ch)
                nb_fp = _get_fp(y_pred_ch, y_true_ch)
                nb_fn = _get_fn(y_pred_ch, y_true_ch)
                _precision = nb_tp / (nb_tp + nb_fp + esp)
                _recall = nb_tp / (nb_tp + nb_fn + esp)
                performs[int(ch)] = 2 * _precision * \
                    _recall / (_precision + _recall + esp)
            mperforms = sum([i*j for (i, j) in zip(performs, weights)])
        return mperforms, performs

class Kappa(object):
    def __init__(self, des="Kappa"):
        self.des = des

    def __repr__(self):
        return "Kapp"

    def __call__(self, y_pred, y_true, threshold=0.5):

        """
        args:
            y_true : 4-d ndarray in [batch_size, chs, img_rows, img_cols]
            y_pred : 4-d ndarray in [batch_size, chs, img_rows, img_cols]
            threshold : [0.0, 1.0]
        return (Po-Pe)/(1-Pe)
        """
        batch_size, chs, img_rows, img_cols = y_true.shape
        device = y_true.device
        if chs == 1:
            y_pred = _binarize(y_pred, threshold)
            y_true = _binarize(y_true, threshold)
            nb_tp = _get_tp(y_pred, y_true)
            nb_fp = _get_fp(y_pred, y_true)
            nb_tn = _get_tn(y_pred, y_true)
            nb_fn = _get_fn(y_pred, y_true)
            nb_total = nb_tp + nb_fp + nb_tn + nb_fn
            Po = (nb_tp + nb_tn) / nb_total
            Pe = ((nb_tp + nb_fp) * (nb_tp + nb_fn) +
                  (nb_fn + nb_tn) * (nb_fp + nb_tn)) / (nb_total**2)
            mperforms = (Po - Pe) / (1 - Pe + esp)
            performs = None
        else:
            y_pred = _argmax(y_pred, 1)
            y_true = _argmax(y_true, 1)
            performs = torch.zeros(chs, 1).to(device)
            weights = _get_weights(y_true, chs)
            for ch in range(chs):
                y_true_ch = torch.zeros(batch_size, img_rows, img_cols)
                y_pred_ch = torch.zeros(batch_size, img_rows, img_cols)
                y_true_ch[y_true == ch] = 1
                y_pred_ch[y_pred == ch] = 1
                nb_tp = _get_tp(y_pred_ch, y_true_ch)
                nb_fp = _get_fp(y_pred_ch, y_true_ch)
                nb_tn = _get_tn(y_pred_ch, y_true_ch)
                nb_fn = _get_fn(y_pred_ch, y_true_ch)
                nb_total = nb_tp + nb_fp + nb_tn + nb_fn
                Po = (nb_tp + nb_tn) / nb_total
                Pe = ((nb_tp + nb_fp) * (nb_tp + nb_fn)
                      + (nb_fn + nb_tn) * (nb_fp + nb_tn)) / (nb_total**2)
                performs[int(ch)] = (Po - Pe) / (1 - Pe + esp)
            mperforms = sum([i*j for (i, j) in zip(performs, weights)])
        return mperforms, performs

class Jaccard(object):
    def __init__(self, des="Jaccard"):
        self.des = des

    def __repr__(self):
        return "Jacc"

    def __call__(self, y_pred, y_true, threshold=0.5):
        """
        args:
            y_true : 4-d ndarray in [batch_size, chs, img_rows, img_cols]
            y_pred : 4-d ndarray in [batch_size, chs, img_rows, img_cols]
            threshold : [0.0, 1.0]
        return intersection / (sum-intersection)
        """
        batch_size, chs, img_rows, img_cols = y_true.shape
        device = y_true.device
        if chs == 1:
            y_pred = _binarize(y_pred, threshold)
            y_true = _binarize(y_true, threshold)
            _intersec = torch.sum(y_true * y_pred).float()
            _sum = torch.sum(y_true + y_pred).float()
            mperforms = _intersec / (_sum - _intersec + esp)
            performs = None
        else:
            y_pred = _argmax(y_pred, 1)
            y_true = _argmax(y_true, 1)
            performs = torch.zeros(chs, 1).to(device)
            weights = _get_weights(y_true, chs)
            for ch in range(chs):
                y_true_ch = torch.zeros(batch_size, img_rows, img_cols)
                y_pred_ch = torch.zeros(batch_size, img_rows, img_cols)
                y_true_ch[y_true == ch] = 1
                y_pred_ch[y_pred == ch] = 1
                _intersec = torch.sum(y_true_ch * y_pred_ch).float()
                _sum = torch.sum(y_true_ch + y_pred_ch).float()
                performs[int(ch)] = _intersec / (_sum - _intersec + esp)
            mperforms = sum([i*j for (i, j) in zip(performs, weights)])
        return mperforms, performs

class MSE(object):
    def __init__(self, des="Mean Square Error"):
        self.des = des

    def __repr__(self):
        return "MSE"

    def __call__(self, y_pred, y_true, dim=1, threshold=None):
        """
        args:
            y_true : 4-d ndarray in [batch_size, channels, img_rows, img_cols]
            y_pred : 4-d ndarray in [batch_size, channels, img_rows, img_cols]
            threshold : [0.0, 1.0]
        return mean_squared_error, smaller the better
        """
        if threshold:
            y_pred = _binarize(y_pred, threshold)
        return torch.mean((y_pred - y_true) ** 2)

class PSNR(object):
    def __init__(self, des="Peak Signal to Noise Ratio"):
        self.des = des

    def __repr__(self):
        return "PSNR"

    def __call__(self, y_pred, y_true, dim=1, threshold=None):
        """
        args:
            y_true : 4-d ndarray in [batch_size, channels, img_rows, img_cols]
            y_pred : 4-d ndarray in [batch_size, channels, img_rows, img_cols]
            threshold : [0.0, 1.0]
        return PSNR, larger the better
        """
        if threshold:
            y_pred = _binarize(y_pred, threshold)
        mse = torch.mean((y_pred - y_true) ** 2)
        return 10 * torch.log10(1 / mse)

class SSIM(object):
    '''
    modified from https://github.com/jorge-pessoa/pytorch-msssim
    '''
    def __init__(self, des="structural similarity index"):
        self.des = des

    def __repr__(self):
        return "SSIM"

    def gaussian(self, w_size, sigma):
        gauss = torch.Tensor([math.exp(-(x - w_size//2)**2/float(2*sigma**2)) for x in range(w_size)])
        return gauss/gauss.sum()

    def create_window(self, w_size, channel=1):
        _1D_window = self.gaussian(w_size, 1.5).unsqueeze(1)
        _2D_window = _1D_window.mm(_1D_window.t()).float().unsqueeze(0).unsqueeze(0)
        window = _2D_window.expand(channel, 1, w_size, w_size).contiguous()
        return window

    def __call__(self, y_pred, y_true, w_size=11, size_average=True, full=False):
        """
        args:
            y_true : 4-d ndarray in [batch_size, channels, img_rows, img_cols]
            y_pred : 4-d ndarray in [batch_size, channels, img_rows, img_cols]
            w_size : int, default 11
            size_average : boolean, default True
            full : boolean, default False
        return ssim, larger the better
        """
        # Value range can be different from 255. Other common ranges are 1 (sigmoid) and 2 (tanh).
        if torch.max(y_pred) > 128:
            max_val = 255
        else:
            max_val = 1

        if torch.min(y_pred) < -0.5:
            min_val = -1
        else:
            min_val = 0
        L = max_val - min_val

        padd = 0
        (_, channel, height, width) = y_pred.size()
        window = self.create_window(w_size, channel=channel).to(y_pred.device)

        mu1 = F.conv2d(y_pred, window, padding=padd, groups=channel)
        mu2 = F.conv2d(y_true, window, padding=padd, groups=channel)

        mu1_sq = mu1.pow(2)
        mu2_sq = mu2.pow(2)
        mu1_mu2 = mu1 * mu2

        sigma1_sq = F.conv2d(y_pred * y_pred, window, padding=padd, groups=channel) - mu1_sq
        sigma2_sq = F.conv2d(y_true * y_true, window, padding=padd, groups=channel) - mu2_sq
        sigma12 = F.conv2d(y_pred * y_true, window, padding=padd, groups=channel) - mu1_mu2

        C1 = (0.01 * L) ** 2
        C2 = (0.03 * L) ** 2

        v1 = 2.0 * sigma12 + C2
        v2 = sigma1_sq + sigma2_sq + C2
        cs = torch.mean(v1 / v2)  # contrast sensitivity

        ssim_map = ((2 * mu1_mu2 + C1) * v1) / ((mu1_sq + mu2_sq + C1) * v2)

        if size_average:
            ret = ssim_map.mean()
        else:
            ret = ssim_map.mean(1).mean(1).mean(1)

        if full:
            return ret, cs
        return ret

class AE(object):
    """
    Modified from matlab : colorangle.m, MATLAB V2019b
    angle = acos(RGB1' * RGB2 / (norm(RGB1) * norm(RGB2)));
    angle = 180 / pi * angle;
    """
    def __init__(self, des='average Angular Error'):
        self.des = des

    def __repr__(self):
        return "AE"

    def __call__(self, y_pred, y_true):
        """
        args:
            y_true : 4-d ndarray in [batch_size, channels, img_rows, img_cols]
            y_pred : 4-d ndarray in [batch_size, channels, img_rows, img_cols]
        return average AE, smaller the better
        """
        dotP = torch.sum(y_pred * y_true, dim=1)
        Norm_pred = torch.sqrt(torch.sum(y_pred * y_pred, dim=1))
        Norm_true = torch.sqrt(torch.sum(y_true * y_true, dim=1))
        ae = 180 / math.pi * torch.acos(dotP / (Norm_pred * Norm_true + eps))
        return ae.mean(1).mean(1)

if __name__ == "__main__":
    for ch in [3, 1]:
        batch_size, img_row, img_col = 1, 224, 224
        y_true = torch.rand(batch_size, ch, img_row, img_col)
        noise = torch.zeros(y_true.size()).data.normal_(0, std=0.1)
        y_pred = y_true + noise
        for cuda in [False, True]:
            if cuda:
                y_pred = y_pred.cuda()
                y_true = y_true.cuda()

            print('#'*20, 'Cuda : {} ; size : {}'.format(cuda, y_true.size()))
            ########### similarity metrics
            metric = MSE()
            acc = metric(y_pred, y_true).item()
            print("{} ==> {}".format(repr(metric), acc))

            metric = PSNR()
            acc = metric(y_pred, y_true).item()
            print("{} ==> {}".format(repr(metric), acc))

            metric = SSIM()
            acc = metric(y_pred, y_true).item()
            print("{} ==> {}".format(repr(metric), acc))

            metric = LPIPS(cuda)
            acc = metric(y_pred, y_true).item()
            print("{} ==> {}".format(repr(metric), acc))

            met
# ... truncated ...


#!/usr/bin/env python
# -*- coding: UTF-8 -*-
#copyright (c) go-hiroaki & Chokurei
#email: guangmingwu2010@gmail.com
#       guozhilingty@gmail.com
#
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
import math
import torch
import torch.nn as nn
import torch.nn.functional as F

eps = 1e-6

def _binarize(y_data, threshold):
    """
    args:
        y_data : [float] 4-d tensor in [batch_size, channels, img_rows, img_cols]
        threshold : [float] [0.0, 1.0]
    return 4-d binarized y_data
    """
    y_data[y_data < threshold] = 0.0
    y_data[y_data >= threshold] = 1.0
    return y_data

def _argmax(y_data, dim):
    """
    args:
        y_data : 4-d tensor in [batch_size, chs, img_rows, img_cols]
        dim : int
    return 3-d [int] y_data
    """
    return torch.argmax(y_data, dim).int()

def _get_tp(y_pred, y_true):
    """
    args:
        y_true : [int] 3-d in [batch_size, img_rows, img_cols]
        y_pred : [int] 3-d in [batch_size, img_rows, img_cols]
    return [float] true_positive
    """
    return torch.sum(y_true * y_pred).float()


# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
import math
import torch
import torch.nn as nn
import torch.nn.functional as F

eps = 1e-6

def _binarize(y_data, threshold):
    """
    args:
        y_data : [float] 4-d tensor in [batch_size, channels, img_rows, img_cols]
        threshold : [float] [0.0, 1.0]
    return 4-d binarized y_data
    """
    y_data[y_data < threshold] = 0.0
    y_data[y_data >= threshold] = 1.0
    return y_data

def _argmax(y_data, dim):
    """
    args:
        y_data : 4-d tensor in [batch_size, chs, img_rows, img_cols]
        dim : int
    return 3-d [int] y_data
    """
    return torch.argmax(y_data, dim).int()

def _get_tp(y_pred, y_true):
    """
    args:
        y_true : [int] 3-d in [batch_size, img_rows, img_cols]
        y_pred : [int] 3-d in [batch_size, img_rows, img_cols]
    return [float] true_positive
    """
    return torch.sum(y_true * y_pred).float()

def _get_fp(y_pred, y_true):
    """
    args:
        y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
        y_pred : 3-d ndarray in [batch_size, img_rows, img_cols]
    return [float] false_positive
    """
    return torch.sum((1 - y_true) * y_pred).float()


    """
    args:
        y_data : [float] 4-d tensor in [batch_size, channels, img_rows, img_cols]
        threshold : [float] [0.0, 1.0]
    return 4-d binarized y_data
    """
    y_data[y_data < threshold] = 0.0
    y_data[y_data >= threshold] = 1.0
    return y_data

def _argmax(y_data, dim):
    """
    args:
        y_data : 4-d tensor in [batch_size, chs, img_rows, img_cols]
        dim : int
    return 3-d [int] y_data
    """
    return torch.argmax(y_data, dim).int()

def _get_tp(y_pred, y_true):
    """
    args:
        y_true : [int] 3-d in [batch_size, img_rows, img_cols]
        y_pred : [int] 3-d in [batch_size, img_rows, img_cols]
    return [float] true_positive
    """
    return torch.sum(y_true * y_pred).float()

def _get_fp(y_pred, y_true):
    """
    args:
        y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
        y_pred : 3-d ndarray in [batch_size, img_rows, img_cols]
    return [float] false_positive
    """
    return torch.sum((1 - y_true) * y_pred).float()

def _get_tn(y_pred, y_true):
    """
    args:
        y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
        y_pred : 3-d ndarray in [batch_size, img_rows, img_cols]
    return [float] true_negative
    """
    return torch.sum((1 - y_true) * (1 - y_pred)).float()


def _argmax(y_data, dim):
    """
    args:
        y_data : 4-d tensor in [batch_size, chs, img_rows, img_cols]
        dim : int
    return 3-d [int] y_data
    """
    return torch.argmax(y_data, dim).int()

def _get_tp(y_pred, y_true):
    """
    args:
        y_true : [int] 3-d in [batch_size, img_rows, img_cols]
        y_pred : [int] 3-d in [batch_size, img_rows, img_cols]
    return [float] true_positive
    """
    return torch.sum(y_true * y_pred).float()

def _get_fp(y_pred, y_true):
    """
    args:
        y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
        y_pred : 3-d ndarray in [batch_size, img_rows, img_cols]
    return [float] false_positive
    """
    return torch.sum((1 - y_true) * y_pred).float()

def _get_tn(y_pred, y_true):
    """
    args:
        y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
        y_pred : 3-d ndarray in [batch_size, img_rows, img_cols]
    return [float] true_negative
    """
    return torch.sum((1 - y_true) * (1 - y_pred)).float()

def _get_fn(y_pred, y_true):
    """
    args:
        y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
        y_pred : 3-d ndarray in [batch_size, img_rows, img_cols]
    return [float] false_negative
    """
    return torch.sum(y_true * (1 - y_pred)).float()


def _get_tp(y_pred, y_true):
    """
    args:
        y_true : [int] 3-d in [batch_size, img_rows, img_cols]
        y_pred : [int] 3-d in [batch_size, img_rows, img_cols]
    return [float] true_positive
    """
    return torch.sum(y_true * y_pred).float()

def _get_fp(y_pred, y_true):
    """
    args:
        y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
        y_pred : 3-d ndarray in [batch_size, img_rows, img_cols]
    return [float] false_positive
    """
    return torch.sum((1 - y_true) * y_pred).float()

def _get_tn(y_pred, y_true):
    """
    args:
        y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
        y_pred : 3-d ndarray in [batch_size, img_rows, img_cols]
    return [float] true_negative
    """
    return torch.sum((1 - y_true) * (1 - y_pred)).float()

def _get_fn(y_pred, y_true):
    """
    args:
        y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
        y_pred : 3-d ndarray in [batch_size, img_rows, img_cols]
    return [float] false_negative
    """
    return torch.sum(y_true * (1 - y_pred)).float()

def _get_weights(y_true, nb_ch):
    """
    args:
        y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
        nb_ch : int
    return [float] weights
    """
    batch_size, img_rows, img_cols = y_true.shape


def _get_fp(y_pred, y_true):
    """
    args:
        y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
        y_pred : 3-d ndarray in [batch_size, img_rows, img_cols]
    return [float] false_positive
    """
    return torch.sum((1 - y_true) * y_pred).float()

def _get_tn(y_pred, y_true):
    """
    args:
        y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
        y_pred : 3-d ndarray in [batch_size, img_rows, img_cols]
    return [float] true_negative
    """
    return torch.sum((1 - y_true) * (1 - y_pred)).float()

def _get_fn(y_pred, y_true):
    """
    args:
        y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
        y_pred : 3-d ndarray in [batch_size, img_rows, img_cols]
    return [float] false_negative
    """
    return torch.sum(y_true * (1 - y_pred)).float()

def _get_weights(y_true, nb_ch):
    """
    args:
        y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
        nb_ch : int
    return [float] weights
    """
    batch_size, img_rows, img_cols = y_true.shape
    pixels = batch_size * img_rows * img_cols
    weights = [torch.sum(y_true==ch).item() / pixels for ch in range(nb_ch)]
    return weights

class CFMatrix(object):
    def __init__(self, des=None):
        self.des = des

    def __repr__(self):


def _get_tn(y_pred, y_true):
    """
    args:
        y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
        y_pred : 3-d ndarray in [batch_size, img_rows, img_cols]
    return [float] true_negative
    """
    return torch.sum((1 - y_true) * (1 - y_pred)).float()

def _get_fn(y_pred, y_true):
    """
    args:
        y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
        y_pred : 3-d ndarray in [batch_size, img_rows, img_cols]
    return [float] false_negative
    """
    return torch.sum(y_true * (1 - y_pred)).float()

def _get_weights(y_true, nb_ch):
    """
    args:
        y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
        nb_ch : int
    return [float] weights
    """
    batch_size, img_rows, img_cols = y_true.shape
    pixels = batch_size * img_rows * img_cols
    weights = [torch.sum(y_true==ch).item() / pixels for ch in range(nb_ch)]
    return weights

class CFMatrix(object):
    def __init__(self, des=None):
        self.des = des

    def __repr__(self):
        return "ConfusionMatrix"

    def __call__(self, y_pred, y_true, ignore_index, threshold=0.5):

        """
        args:
            y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
            y_pred : 4-d ndarray in [batch_size, chs, img_rows, img_cols]
            threshold : [0.0, 1.0]
        return confusion matrix
        """
        batch_size, img_rows, img_cols = y_pred.shape
        chs = ignore_index


CFMatrix

        y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
        y_pred : 3-d ndarray in [batch_size, img_rows, img_cols]
    return [float] false_negative
    """
    return torch.sum(y_true * (1 - y_pred)).float()

def _get_weights(y_true, nb_ch):
    """
    args:
        y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
        nb_ch : int
    return [float] weights
    """
    batch_size, img_rows, img_cols = y_true.shape
    pixels = batch_size * img_rows * img_cols
    weights = [torch.sum(y_true==ch).item() / pixels for ch in range(nb_ch)]
    return weights

class CFMatrix(object):
    def __init__(self, des=None):
        self.des = des

    def __repr__(self):
        return "ConfusionMatrix"

    def __call__(self, y_pred, y_true, ignore_index, threshold=0.5):

        """
        args:
            y_true : 3-d ndarray in [batch_size, img_rows, img_cols]
            y_pred : 4-d ndarray in [batch_size, chs, img_rows, img_cols]
            threshold : [0.0, 1.0]
        return confusion matrix
        """
        batch_size, img_rows, img_cols = y_pred.shape
        chs = ignore_index
        device = y_true.device
        if chs == 1:
            y_pred = _binarize(y_pred, threshold)
            y_true = _binarize(y_true, threshold)
            nb_tp = _get_tp(y_pred, y_true)
            nb_fp = _get_fp(y_pred, y_true)
            nb_tn = _get_tn(y_pred, y_true)
            nb_fn = _get_fn(y_pred, y_true)
            mperforms = [nb_tp, nb_fp, nb_tn, nb_fn]
            performs = None
        else:
            performs = torch.zeros(chs, 4).to(device)
            weights = _get_weights(y_true, chs)
            for ch in range(chs):
                y_true_ch = torch.zeros(batch_size, img_rows, img_cols)
                y_false_ch = torch.zeros(batch_size, img_rows, img_cols)
                y_pred_ch = torch.zeros(batch_size, img_rows, img_cols)
                y_true_ch[y_true == ch] = 1
                y_false_ch[torch.logical_and((y_true != ch), (y_true != ignore_index))] = 1
                y_pred_ch[y_pred == ch] = 1
                nb_tp = _get_tp(y_pred_ch, y_true_ch)
                nb_fp = torch.sum(y_false_ch * y_pred_ch).float()
                nb_tn = torch.sum(y_false_ch * (1 - y_pred_ch)).float()
                nb_fn = _get_fn(y_pred_ch, y_true_ch)
                performs[int(ch), :] = torch.FloatTensor([nb_tp, nb_fp, nb_tn, nb_fn])
            mperforms = sum([i*j for (i, j) in zip(performs, weights)])
        return mperforms, performs

class OAAcc(object):
    def __init__(self, des="Overall Accuracy"):
        self.des = des

    def __repr__(self):
        return "OAcc"

    def __call__(self, y_pred, y_true, threshold=0.5):
        """
        args:
            y_true : 4-d ndarray in [batch_size, chs, img_rows, img_cols]
            y_pred : 4-d ndarray in [batch_size, chs, img_rows, img_cols]
            threshold : [0.0, 1.0]
        return (tp+tn)/total
        """
        batch_size, chs, img_rows, img_cols = y_true.shape
        device = y_true.device
        if chs == 1: