mmocr/core/evaluation/hmean.py

# Copyright (c) OpenMMLab. All rights reserved.
from operator import itemgetter

import mmcv
from mmcv.utils import print_log

import mmocr.utils as utils
from mmocr.core.evaluation import hmean_ic13, hmean_iou
from mmocr.core.evaluation.utils import (filter_2dlist_result,
                                         select_top_boundary)
from mmocr.core.mask import extract_boundary


def output_ranklist(img_results, img_infos, out_file):
    """Output the worst results for debugging.

    Args:
        img_results (list[dict]): Image result list.
        img_infos (list[dict]): Image information list.
        out_file (str): The output file path.

    Returns:
        sorted_results (list[dict]): Image results sorted by hmean.
    """
    assert utils.is_type_list(img_results, dict)
    assert utils.is_type_list(img_infos, dict)
    assert isinstance(out_file, str)
    assert out_file.endswith('json')

    sorted_results = []
    for idx, result in enumerate(img_results):
        name = img_infos[idx]['file_name']
        img_result = result
        img_result['file_name'] = name
        sorted_results.append(img_result)
    sorted_results = sorted(
        sorted_results, key=itemgetter('hmean'), reverse=False)

    mmcv.dump(sorted_results, file=out_file)

    return sorted_results


def get_gt_masks(ann_infos):
    """Get ground truth masks and ignored masks.

    Args:
        ann_infos (list[dict]): Each dict contains annotation
            infos of one image, containing following keys:
            masks, masks_ignore.
    Returns:
        gt_masks (list[list[list[int]]]): Ground truth masks.
        gt_masks_ignore (list[list[list[int]]]): Ignored masks.
    """
    assert utils.is_type_list(ann_infos, dict)

    gt_masks = []
    gt_masks_ignore = []
    for ann_info in ann_infos:
        masks = ann_info['masks']
        mask_gt = []
        for mask in masks:
            assert len(mask[0]) >= 8 and len(mask[0]) % 2 == 0
            mask_gt.append(mask[0])
        gt_masks.append(mask_gt)

        masks_ignore = ann_info['masks_ignore']
        mask_gt_ignore = []
        for mask_ignore in masks_ignore:
            assert len(mask_ignore[0]) >= 8 and len(mask_ignore[0]) % 2 == 0
            mask_gt_ignore.append(mask_ignore[0])
        gt_masks_ignore.append(mask_gt_ignore)

    return gt_masks, gt_masks_ignore


def eval_hmean(results,
               img_infos,
               ann_infos,
               metrics={'hmean-iou'},
               score_thr=0.3,
               rank_list=None,
               logger=None,
               **kwargs):
    """Evaluation in hmean metric.

    Args:
        results (list[dict]): Each dict corresponds to one image,
            containing the following keys: boundary_result
        img_infos (list[dict]): Each dict corresponds to one image,
            containing the following keys: filename, height, width
        ann_infos (list[dict]): Each dict corresponds to one image,
            containing the following keys: masks, masks_ignore
        score_thr (float): Score threshold of prediction map.
        metrics (set{str}): Hmean metric set, should be one or all of
            {'hmean-iou', 'hmean-ic13'}
    Returns:
        dict[str: float]
    """
    assert utils.is_type_list(results, dict)
    assert utils.is_type_list(img_infos, dict)
    assert utils.is_type_list(ann_infos, dict)
    assert len(results) == len(img_infos) == len(ann_infos)
    assert isinstance(metrics, set)

    gts, gts_ignore = get_gt_masks(ann_infos)

    preds = []
    pred_scores = []
    for result in results:
        _, texts, scores = extract_boundary(result)
        if len(texts) > 0:
            assert utils.valid_boundary(texts[0], False)
        valid_texts, valid_text_scores = filter_2dlist_result(
            texts, scores, score_thr)
        preds.append(valid_texts)
        pred_scores.append(valid_text_scores)

    eval_results = {}
    for metric in metrics:
        msg = f'Evaluating {metric}...'
        if logger is None:
            msg = '\n' + msg
        print_log(msg, logger=logger)
        best_result = dict(hmean=-1)
        for iter in range(3, 10):
            thr = iter * 0.1
            if thr < score_thr:
                continue
            top_preds = select_top_boundary(preds, pred_scores, thr)
            if metric == 'hmean-iou':
                result, img_result = hmean_iou.eval_hmean_iou(
                    top_preds, gts, gts_ignore)
            elif metric == 'hmean-ic13':
                result, img_result = hmean_ic13.eval_hmean_ic13(
                    top_preds, gts, gts_ignore)
            else:
                raise NotImplementedError
            if rank_list is not None:
                output_ranklist(img_result, img_infos, rank_list)

            print_log(
                'thr {0:.2f}, recall: {1[recall]:.3f}, '
                'precision: {1[precision]:.3f}, '
                'hmean: {1[hmean]:.3f}'.format(thr, result),
                logger=logger)
            if result['hmean'] > best_result['hmean']:
                best_result = result
        eval_results[metric + ':recall'] = best_result['recall']
        eval_results[metric + ':precision'] = best_result['precision']
        eval_results[metric + ':hmean'] = best_result['hmean']
    return eval_results