efficientdet/effdet/evaluation/np_mask_list.py

import numpy as np
from .np_box_list import *

EPSILON = 1e-7


class MaskList(BoxList):
    """Convenience wrapper for BoxList with masks.
  
    BoxMaskList extends the np_box_list.BoxList to contain masks as well.
    In particular, its constructor receives both boxes and masks. Note that the
    masks correspond to the full image.
    """

    def __init__(self, box_data, mask_data):
        """Constructs box collection.
    
        Args:
            box_data: a numpy array of shape [N, 4] representing box coordinates
            mask_data: a numpy array of shape [N, height, width] representing masks
                with values are in {0,1}. The masks correspond to the full
                image. The height and the width will be equal to image height and width.
    
        Raises:
            ValueError: if bbox data is not a numpy array
            ValueError: if invalid dimensions for bbox data
            ValueError: if mask data is not a numpy array
            ValueError: if invalid dimension for mask data
        """
        super(MaskList, self).__init__(box_data)
        if not isinstance(mask_data, np.ndarray):
            raise ValueError('Mask data must be a numpy array.')
        if len(mask_data.shape) != 3:
            raise ValueError('Invalid dimensions for mask data.')
        if mask_data.dtype != np.uint8:
            raise ValueError('Invalid data type for mask data: uint8 is required.')
        if mask_data.shape[0] != box_data.shape[0]:
            raise ValueError('There should be the same number of boxes and masks.')
        self.data['masks'] = mask_data

    def get_masks(self):
        """Convenience function for accessing masks.
    
        Returns:
            a numpy array of shape [N, height, width] representing masks
        """
        return self.get_field('masks')


def boxlist_to_masklist(boxlist):
    """Converts a BoxList containing 'masks' into a BoxMaskList.
  
    Args:
        boxlist: An np_box_list.BoxList object.
  
    Returns:
        An BoxMaskList object.
  
    Raises:
        ValueError: If boxlist does not contain `masks` as a field.
    """
    if not boxlist.has_field('masks'):
        raise ValueError('boxlist does not contain mask field.')
    masklist = MaskList(box_data=boxlist.get(), mask_data=boxlist.get_field('masks'))
    extra_fields = boxlist.get_extra_fields()
    for key in extra_fields:
        if key != 'masks':
            masklist.data[key] = boxlist.get_field(key)
    return masklist


def area_mask(masks):
    """Computes area of masks.

    Args:
        masks: Numpy array with shape [N, height, width] holding N masks. Masks
        values are of type np.uint8 and values are in {0,1}.

    Returns:
        a numpy array with shape [N*1] representing mask areas.

    Raises:
        ValueError: If masks.dtype is not np.uint8
    """
    if masks.dtype != np.uint8:
        raise ValueError('Masks type should be np.uint8')
    return np.sum(masks, axis=(1, 2), dtype=np.float32)


def intersection_mask(masks1, masks2):
    """Compute pairwise intersection areas between masks.

    Args:
        masks1: a numpy array with shape [N, height, width] holding N masks. Masks
            values are of type np.uint8 and values are in {0,1}.
        masks2: a numpy array with shape [M, height, width] holding M masks. Masks
            values are of type np.uint8 and values are in {0,1}.

    Returns:
        a numpy array with shape [N*M] representing pairwise intersection area.

    Raises:
        ValueError: If masks1 and masks2 are not of type np.uint8.
    """
    if masks1.dtype != np.uint8 or masks2.dtype != np.uint8:
        raise ValueError('masks1 and masks2 should be of type np.uint8')
    n = masks1.shape[0]
    m = masks2.shape[0]
    answer = np.zeros([n, m], dtype=np.float32)
    for i in np.arange(n):
        for j in np.arange(m):
            answer[i, j] = np.sum(np.minimum(masks1[i], masks2[j]), dtype=np.float32)
    return answer


def iou_mask(masks1, masks2):
    """Computes pairwise intersection-over-union between mask collections.

    Args:
        masks1: a numpy array with shape [N, height, width] holding N masks. Masks
            values are of type np.uint8 and values are in {0,1}.
        masks2: a numpy array with shape [M, height, width] holding N masks. Masks
            values are of type np.uint8 and values are in {0,1}.

    Returns:
        a numpy array with shape [N, M] representing pairwise iou scores.

    Raises:
        ValueError: If masks1 and masks2 are not of type np.uint8.
    """
    if masks1.dtype != np.uint8 or masks2.dtype != np.uint8:
        raise ValueError('masks1 and masks2 should be of type np.uint8')
    intersect = intersection(masks1, masks2)
    area1 = area(masks1)
    area2 = area(masks2)
    union = np.expand_dims(area1, axis=1) + np.expand_dims(area2, axis=0) - intersect
    return intersect / np.maximum(union, EPSILON)


def ioa_mask(masks1, masks2):
    """Computes pairwise intersection-over-area between box collections.

    Intersection-over-area (ioa) between two masks, mask1 and mask2 is defined as
    their intersection area over mask2's area. Note that ioa is not symmetric,
    that is, IOA(mask1, mask2) != IOA(mask2, mask1).

    Args:
        masks1: a numpy array with shape [N, height, width] holding N masks. Masks
            values are of type np.uint8 and values are in {0,1}.
        masks2: a numpy array with shape [M, height, width] holding N masks. Masks
            values are of type np.uint8 and values are in {0,1}.

    Returns:
        a numpy array with shape [N, M] representing pairwise ioa scores.

    Raises:
        ValueError: If masks1 and masks2 are not of type np.uint8.
    """
    if masks1.dtype != np.uint8 or masks2.dtype != np.uint8:
        raise ValueError('masks1 and masks2 should be of type np.uint8')
    intersect = intersection(masks1, masks2)
    areas = np.expand_dims(area(masks2), axis=0)
    return intersect / (areas + EPSILON)


def area_masklist(masklist):
    """Computes area of masks.
  
    Args:
        masklist: BoxMaskList holding N boxes and masks
  
    Returns:
        a numpy array with shape [N*1] representing mask areas
    """
    return area_mask(masklist.get_masks())


def intersection_masklist(masklist1, masklist2):
    """Compute pairwise intersection areas between masks.
  
    Args:
        masklist1: BoxMaskList holding N boxes and masks
        masklist2: BoxMaskList holding M boxes and masks
  
    Returns:
        a numpy array with shape [N*M] representing pairwise intersection area
    """
    return intersection_mask(masklist1.get_masks(), masklist2.get_masks())


def iou_masklist(masklist1, masklist2):
    """Computes pairwise intersection-over-union between box and mask collections.
  
    Args:
        masklist1: BoxMaskList holding N boxes and masks
        masklist2: BoxMaskList holding M boxes and masks
  
    Returns:
        a numpy array with shape [N, M] representing pairwise iou scores.
    """
    return iou_mask(masklist1.get_masks(), masklist2.get_masks())


def ioa_masklist(masklist1, masklist2):
    """Computes pairwise intersection-over-area between box and mask collections.
  
    Intersection-over-area (ioa) between two masks mask1 and mask2 is defined as
    their intersection area over mask2's area. Note that ioa is not symmetric,
    that is, IOA(mask1, mask2) != IOA(mask2, mask1).
  
    Args:
        masklist1: BoxMaskList holding N boxes and masks
        masklist2: BoxMaskList holding M boxes and masks
  
    Returns:
        a numpy array with shape [N, M] representing pairwise ioa scores.
    """
    return ioa_mask(masklist1.get_masks(), masklist2.get_masks())


def gather_masklist(masklist, indices, fields=None):
    """Gather boxes from BoxMaskList according to indices.
  
    By default, gather returns boxes corresponding to the input index list, as
    well as all additional fields stored in the masklist (indexing into the
    first dimension).  However one can optionally only gather from a
    subset of fields.
  
    Args:
        masklist: BoxMaskList holding N boxes
        indices: a 1-d numpy array of type int_
        fields: (optional) list of fields to also gather from.  If None (default), all fields
            are gathered from.  Pass an empty fields list to only gather the box coordinates.
  
    Returns:
        submasklist: a BoxMaskList corresponding to the subset of the input masklist specified by indices
  
    Raises:
        ValueError: if specified field is not contained in masklist or if the indices are not of type int_
    """
    if fields is not None:
        if 'masks' not in fields:
            fields.append('masks')
    return boxlist_to_masklist(gather_boxlist(boxlist=masklist, indices=indices, fields=fields))


def sort_by_field_masklist(masklist, field, order=SortOrder.DESCEND):
    """Sort boxes and associated fields according to a scalar field.
  
    A common use case is reordering the boxes according to descending scores.
  
    Args:
        masklist: BoxMaskList holding N boxes.
        field: A BoxMaskList field for sorting and reordering the BoxMaskList.
        order: (Optional) 'descend' or 'ascend'. Default is descend.
  
    Returns:
        sorted_masklist: A sorted BoxMaskList with the field in the specified order.
    """
    return boxlist_to_masklist(sort_by_field_boxlist(boxlist=masklist, field=field, order=order))


def non_max_suppression_mask(masklist, max_output_size=10000, iou_threshold=1.0, score_threshold=-10.0):
    """Non maximum suppression.
  
    This op greedily selects a subset of detection bounding boxes, pruning
    away boxes that have high IOU (intersection over union) overlap (> thresh)
    with already selected boxes. In each iteration, the detected bounding box with
    highest score in the available pool is selected.
  
    Args:
        masklist: BoxMaskList holding N boxes.  Must contain a 'scores' field representing
            detection scores. All scores belong to the same class.
        max_output_size: maximum number of retained boxes
        iou_threshold: intersection over union threshold.
        score_threshold: minimum score threshold. Remove the boxes with scores
            less than this value. Default value is set to -10. A very
            low threshold to pass pretty much all the boxes, unless
            the user sets a different score threshold.
  
    Returns:
        an BoxMaskList holding M boxes where M <= max_output_size
  
    Raises:
        ValueError: if 'scores' field does not exist
        ValueError: if threshold is not in [0, 1]
        ValueError: if max_output_size < 0
    """
    if not masklist.has_field('scores'):
        raise ValueError('Field scores does not exist')
    if iou_threshold < 0. or iou_threshold > 1.0:
        raise ValueError('IOU threshold must be in [0, 1]')
    if max_output_size < 0:
        raise ValueError('max_output_size must be bigger than 0.')

    masklist = filter_scores_greater_than(masklist, score_threshold)
    if masklist.num_boxes() == 0:
        return masklist

    masklist = sort_by_field_boxlist(masklist, 'scores')

    # Prevent further computation if NMS is disabled.
    if iou_threshold == 1.0:
        if masklist.num_boxes() > max_output_size:
            selected_indices = np.arange(max_output_size)
            return gather_masklist(masklist, selected_indices)
        else:
            return masklist

    masks = masklist.get_masks()
    num_masks = masklist.num_boxes()

    # is_index_valid is True only for all remaining valid boxes,
    is_index_valid = np.full(num_masks, 1, dtype=bool)
    selected_indices = []
    num_output = 0
    for i in range(num_masks):
        if num_output < max_output_size:
            if is_index_valid[i]:
                num_output += 1
                selected_indices.append(i)
                is_index_valid[i] = False
                valid_indices = np.where(is_index_valid)[0]
                if valid_indices.size == 0:
                    break

                intersect_over_union = iou_mask(np.expand_dims(masks[i], axis=0), masks[valid_indices])
                intersect_over_union = np.squeeze(intersect_over_union, axis=0)
                is_index_valid[valid_indices] = np.logical_and(
                    is_index_valid[valid_indices],
                    intersect_over_union <= iou_threshold)
    return gather_masklist(masklist, np.array(selected_indices))


def multi_class_non_max_suppression_mask(masklist, score_thresh, iou_thresh, max_output_size):
    """Multi-class version of non maximum suppression.
  
    This op greedily selects a subset of detection bounding boxes, pruning away boxes that have
    high IOU (intersection over union) overlap (> thresh) with already selected boxes.  It
    operates independently for each class for which scores are provided (via the scores field
    of the input box_list), pruning boxes with score less than a provided threshold prior to
    applying NMS.
  
    Args:
        masklist: BoxMaskList holding N boxes.  Must contain a 'scores' field representing detection
            scores.  This scores field is a tensor that can be 1 dimensional (in the case of a
            single class) or 2-dimensional, in which case we assume that it takes the shape
            [num_boxes, num_classes]. We further assume that this rank is known statically and
            that scores.shape[1] is also known (i.e., the number of classes is fixed and known
            at graph construction time).
        score_thresh: scalar threshold for score (low scoring boxes are removed).
        iou_thresh: scalar threshold for IOU (boxes that that high IOU overlap with previously
            selected boxes are removed).
        max_output_size: maximum number of retained boxes per class.
  
    Returns:
        a masklist holding M boxes with a rank-1 scores field representing
        corresponding scores for each box with scores sorted in decreasing order
        and a rank-1 classes field representing a class label for each box.
    Raises:
        ValueError: if iou_thresh is not in [0, 1] or if input masklist does not have a valid scores field.
    """
    if not 0 <= iou_thresh <= 1.0:
        raise ValueError('thresh must be between 0 and 1')
    if not isinstance(masklist, MaskList):
        raise ValueError('masklist must be a masklist')
    if not masklist.has_field('scores'):
        raise ValueError('input masklist must have \'scores\' field')
    scores = masklist.get_field('scores')
    if len(scores.shape) == 1:
        scores = np.reshape(scores, [-1, 1])
    elif len(scores.shape) == 2:
        if scores.shape[1] is None:
            raise ValueError('scores field must have statically defined second dimension')
    else:
        raise ValueError('scores field must be of rank 1 or 2')

    num_boxes = masklist.num_boxes()
    num_scores = scores.shape[0]
    num_classes = scores.shape[1]

    if num_boxes != num_scores:
        raise ValueError('Incorrect scores field length: actual vs expected.')

    selected_boxes_list = []
    for class_idx in range(num_classes):
        masklist_and_class_scores = MaskList(box_data=masklist.get(), mask_data=masklist.get_masks())
        class_scores = np.reshape(scores[0:num_scores, class_idx], [-1])
        masklist_and_class_scores.add_field('scores', class_scores)
        masklist_filt = filter_scores_greater_than(masklist_and_class_scores, score_thresh)
        nms_result = non_max_suppression(
            masklist_filt,
            max_output_size=max_output_size,
            iou_threshold=iou_thresh,
            score_threshold=score_thresh)
        nms_result.add_field('classes', np.zeros_like(nms_result.get_field('scores')) + class_idx)
        selected_boxes_list.append(nms_result)
    selected_boxes = concatenate_boxlist(selected_boxes_list)
    sorted_boxes = sort_by_field_boxlist(selected_boxes, 'scores')
    return boxlist_to_masklist(boxlist=sorted_boxes)


def prune_non_overlapping_masklist(masklist1, masklist2, minoverlap=0.0):
    """Prunes the boxes in list1 that overlap less than thresh with list2.
  
    For each mask in masklist1, we want its IOA to be more than minoverlap
    with at least one of the masks in masklist2. If it does not, we remove
    it. If the masks are not full size image, we do the pruning based on boxes.
  
    Args:
        masklist1: BoxMaskList holding N boxes and masks.
        masklist2: BoxMaskList holding M boxes and masks.
        minoverlap: Minimum required overlap between boxes, to count them as overlapping.
  
    Returns:
        A pruned masklist with size [N', 4].
    """
    intersection_over_area = ioa_masklist(masklist2, masklist1)  # [M, N] tensor
    intersection_over_area = np.amax(intersection_over_area, axis=0)  # [N] tensor
    keep_bool = np.greater_equal(intersection_over_area, np.array(minoverlap))
    keep_inds = np.nonzero(keep_bool)[0]
    new_masklist1 = gather_masklist(masklist1, keep_inds)
    return new_masklist1


def concatenate_masklist(masklists, fields=None):
    """Concatenate list of masklists.
  
    This op concatenates a list of input masklists into a larger
    masklist.  It also
    handles concatenation of masklist fields as long as the field tensor
    shapes are equal except for the first dimension.
  
    Args:
        masklists: list of BoxMaskList objects
        fields: optional list of fields to also concatenate.  By default, all
            fields from the first BoxMaskList in the list are included in the concatenation.
  
    Returns:
        a masklist with number of boxes equal to sum([masklist.num_boxes() for masklist in masklist])
    Raises:
        ValueError: if masklists is invalid (i.e., is not a list, is empty, or contains non
            masklist objects), or if requested fields are not contained in all masklists
    """
    if fields is not None:
        if 'masks' not in fields:
            fields.append('masks')
    return boxlist_to_masklist(concatenate_boxlist(boxlists=masklists, fields=fields))


def filter_scores_greater_than_masklist(masklist, thresh):
    """Filter to keep only boxes and masks with score exceeding a given threshold.
  
    This op keeps the collection of boxes and masks whose corresponding scores are
    greater than the input threshold.
  
    Args:
        masklist: BoxMaskList holding N boxes and masks.  Must contain a
            'scores' field representing detection scores.
        thresh: scalar threshold
  
    Returns:
        a BoxMaskList holding M boxes and masks where M <= N
  
    Raises:
        ValueError: if masklist not a BoxMaskList object or if it does not have a scores field
    """
    if not isinstance(masklist, MaskList):
        raise ValueError('masklist must be a BoxMaskList')
    if not masklist.has_field('scores'):
        raise ValueError('input masklist must have \'scores\' field')
    scores = masklist.get_field('scores')
    if len(scores.shape) > 2:
        raise ValueError('Scores should have rank 1 or 2')
    if len(scores.shape) == 2 and scores.shape[1] != 1:
        raise ValueError('Scores should have rank 1 or have shape consistent with [None, 1]')
    high_score_indices = np.reshape(np.where(np.greater(scores, thresh)), [-1]).astype(np.int32)
    return gather_masklist(masklist, high_score_indices)