src/utils/semantic.py

import sys
import hydra
import torch
import os.path as osp
from tqdm import tqdm
from copy import deepcopy
from src.utils.hydra import init_config


src_folder = osp.dirname(osp.dirname(osp.abspath(__file__)))
sys.path.append(src_folder)
sys.path.append(osp.join(src_folder, "dependencies/grid_graph/python/bin"))
sys.path.append(osp.join(src_folder, "dependencies/parallel_cut_pursuit/python/wrappers"))


__all__ = ['compute_semantic_metrics', 'compute_semantic_metrics_s3dis_6fold']


def compute_semantic_metrics(
        model,
        datamodule,
        stage='val',
        verbose=True):
    """Helper function to compute the semantic segmentation metrics of a
    model on a given dataset.
    """
    # Local imports to avoid import loop errors
    from src.data import NAGBatch

    # Pick among train, val, and test datasets. It is important to note
    # that the train dataset produces augmented spherical samples of
    # large scenes, while the val and test dataset
    if stage == 'train':
        dataset = datamodule.train_dataset
        dataloader = datamodule.train_dataloader()
    elif stage == 'val':
        dataset = datamodule.val_dataset
        dataloader = datamodule.val_dataloader()
    elif stage == 'test':
        dataset = datamodule.test_dataset
        dataloader = datamodule.test_dataloader()
    else:
        raise ValueError(f"Unknown stage : {stage}")

    # Prevent `NAGAddKeysTo` from removing attributes to allow
    # visualizing them after model inference
    dataset = _set_attribute_preserving_transforms(dataset)

    # Load a dataset item. This will return the hierarchical partition
    # of an entire tile, within a NAG object
    with torch.no_grad():
        enum = tqdm(dataloader) if verbose else dataloader
        for nag_list in enum:
            nag = NAGBatch.from_nag_list([nag.cuda() for nag in nag_list])

            # Apply on-device transforms on the NAG object. For the
            # train dataset, this will select a spherical sample of the
            # larger tile and apply some data augmentations. For the
            # validation and test datasets, this will prepare an entire
            # tile for inference
            nag = dataset.on_device_transform(nag)

            # NB: we use the "validation_step" protocol here, regardless
            # of the stage the data comes from
            model.validation_step(nag, None)

        # Actions taken from on_validation_epoch_end()
        semantic = deepcopy(model.val_cm)
        model.val_cm.reset()

    if not verbose:
        return semantic

    print(f"mIoU : {semantic.miou().cpu().item()}")
    print(f"OA   : {semantic.oa().cpu().item()}")
    print(f"mAcc : {semantic.macc().cpu().item()}")

    return semantic


def compute_semantic_metrics_s3dis_6fold(
        fold_ckpt,
        experiment_config,
        stage='val',
        verbose=False):
    """Helper function to compute the semantic segmentation metrics of a
    model on a S3DIS 6-fold.

    :param fold_ckpt: dict
        Dictionary with S3DIS fold numbers as keys and checkpoint paths
        as values
    :param experiment_config: str
        Experiment config to use for inference. For instance for S3DIS
        with semantic segmentation: 'semantic/s3dis'
    :param stage: str
    :param verbose: bool
    :return:
    """
    # Local import to avoid import loop errors
    from src.metrics import ConfusionMatrix

    # Very ugly fix to ignore lightning's warning messages about the
    # trainer and modules not being connected
    import warnings
    warnings.filterwarnings("ignore")

    semantic_list = []
    num_classes = None

    for fold, ckpt_path in fold_ckpt.items():

        if verbose:
            print(f"\nFold {fold}")

        # Parse the configs using hydra
        cfg = init_config(overrides=[
            f"experiment={experiment_config}",
            f"datamodule.fold={fold}",
            f"ckpt_path={ckpt_path}"])

        # Instantiate the datamodule
        datamodule = hydra.utils.instantiate(cfg.datamodule)
        datamodule.prepare_data()
        datamodule.setup()

        # Instantiate the model
        model = hydra.utils.instantiate(cfg.model)

        # Load pretrained weights from a checkpoint file
        model = model._load_from_checkpoint(cfg.ckpt_path)
        model = model.eval().cuda()

        # Compute metrics on the fold
        semantic = compute_semantic_metrics(
            model,
            datamodule,
            stage=stage,
            verbose=verbose)

        # Gather some details from the model and datamodule before
        # deleting them
        num_classes = datamodule.train_dataset.num_classes

        del model, datamodule

        # Store the metrics for each fold
        semantic_list.append(semantic)

    # Initialize the 6-fold metrics
    semantic_6fold = ConfusionMatrix(num_classes)

    # Group together per-fold panoptic and semantic results
    for i in range(len(semantic_list)):
        semantic_6fold.confmat += semantic_list[i].confmat.cpu()

    # Print computed the metrics
    print(f"\n6-fold")
    print(f"mIoU : {semantic_6fold.miou().cpu().item()}")
    print(f"OA   : {semantic_6fold.oa().cpu().item()}")
    print(f"mAcc : {semantic_6fold.macc().cpu().item()}")

    return semantic_6fold, semantic_list


def _set_attribute_preserving_transforms(dataset):
    """For the sake of visualization, we require that `NAGAddKeysTo`
    does not remove input `Data` attributes after moving them to
    `Data.x`, so we may visualize them.
    """
    # Local imports to avoid import loop errors
    from src.transforms import NAGAddKeysTo

    for t in dataset.on_device_transform.transforms:
        if isinstance(t, NAGAddKeysTo):
            t.delete_after = False

    return dataset