|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| """Tests for eval_util."""
|
|
|
| from __future__ import absolute_import
|
| from __future__ import division
|
| from __future__ import print_function
|
|
|
| import unittest
|
| from absl.testing import parameterized
|
|
|
| import numpy as np
|
| import six
|
| from six.moves import range
|
| import tensorflow.compat.v1 as tf
|
| from google.protobuf import text_format
|
|
|
| from object_detection import eval_util
|
| from object_detection.core import standard_fields as fields
|
| from object_detection.metrics import coco_evaluation
|
| from object_detection.protos import eval_pb2
|
| from object_detection.utils import test_case
|
| from object_detection.utils import tf_version
|
|
|
|
|
| class EvalUtilTest(test_case.TestCase, parameterized.TestCase):
|
|
|
| def _get_categories_list(self):
|
| return [{'id': 1, 'name': 'person'},
|
| {'id': 2, 'name': 'dog'},
|
| {'id': 3, 'name': 'cat'}]
|
|
|
| def _get_categories_list_with_keypoints(self):
|
| return [{
|
| 'id': 1,
|
| 'name': 'person',
|
| 'keypoints': {
|
| 'left_eye': 0,
|
| 'right_eye': 3
|
| }
|
| }, {
|
| 'id': 2,
|
| 'name': 'dog',
|
| 'keypoints': {
|
| 'tail_start': 1,
|
| 'mouth': 2
|
| }
|
| }, {
|
| 'id': 3,
|
| 'name': 'cat'
|
| }]
|
|
|
| def _make_evaluation_dict(self,
|
| resized_groundtruth_masks=False,
|
| batch_size=1,
|
| max_gt_boxes=None,
|
| scale_to_absolute=False):
|
| input_data_fields = fields.InputDataFields
|
| detection_fields = fields.DetectionResultFields
|
|
|
| image = tf.zeros(shape=[batch_size, 20, 20, 3], dtype=tf.uint8)
|
| if batch_size == 1:
|
| key = tf.constant('image1')
|
| else:
|
| key = tf.constant([str(i) for i in range(batch_size)])
|
| detection_boxes = tf.tile(tf.constant([[[0., 0., 1., 1.]]]),
|
| multiples=[batch_size, 1, 1])
|
| detection_scores = tf.tile(tf.constant([[0.8]]), multiples=[batch_size, 1])
|
| detection_classes = tf.tile(tf.constant([[0]]), multiples=[batch_size, 1])
|
| detection_masks = tf.tile(tf.ones(shape=[1, 1, 20, 20], dtype=tf.float32),
|
| multiples=[batch_size, 1, 1, 1])
|
| num_detections = tf.ones([batch_size])
|
| groundtruth_boxes = tf.constant([[0., 0., 1., 1.]])
|
| groundtruth_classes = tf.constant([1])
|
| groundtruth_instance_masks = tf.ones(shape=[1, 20, 20], dtype=tf.uint8)
|
| original_image_spatial_shapes = tf.constant([[20, 20]], dtype=tf.int32)
|
|
|
| groundtruth_keypoints = tf.constant([[0.0, 0.0], [0.5, 0.5], [1.0, 1.0]])
|
| if resized_groundtruth_masks:
|
| groundtruth_instance_masks = tf.ones(shape=[1, 10, 10], dtype=tf.uint8)
|
|
|
| if batch_size > 1:
|
| groundtruth_boxes = tf.tile(tf.expand_dims(groundtruth_boxes, 0),
|
| multiples=[batch_size, 1, 1])
|
| groundtruth_classes = tf.tile(tf.expand_dims(groundtruth_classes, 0),
|
| multiples=[batch_size, 1])
|
| groundtruth_instance_masks = tf.tile(
|
| tf.expand_dims(groundtruth_instance_masks, 0),
|
| multiples=[batch_size, 1, 1, 1])
|
| groundtruth_keypoints = tf.tile(
|
| tf.expand_dims(groundtruth_keypoints, 0),
|
| multiples=[batch_size, 1, 1])
|
| original_image_spatial_shapes = tf.tile(original_image_spatial_shapes,
|
| multiples=[batch_size, 1])
|
|
|
| detections = {
|
| detection_fields.detection_boxes: detection_boxes,
|
| detection_fields.detection_scores: detection_scores,
|
| detection_fields.detection_classes: detection_classes,
|
| detection_fields.detection_masks: detection_masks,
|
| detection_fields.num_detections: num_detections
|
| }
|
| groundtruth = {
|
| input_data_fields.groundtruth_boxes: groundtruth_boxes,
|
| input_data_fields.groundtruth_classes: groundtruth_classes,
|
| input_data_fields.groundtruth_keypoints: groundtruth_keypoints,
|
| input_data_fields.groundtruth_instance_masks:
|
| groundtruth_instance_masks,
|
| input_data_fields.original_image_spatial_shape:
|
| original_image_spatial_shapes
|
| }
|
| if batch_size > 1:
|
| return eval_util.result_dict_for_batched_example(
|
| image, key, detections, groundtruth,
|
| scale_to_absolute=scale_to_absolute,
|
| max_gt_boxes=max_gt_boxes)
|
| else:
|
| return eval_util.result_dict_for_single_example(
|
| image, key, detections, groundtruth,
|
| scale_to_absolute=scale_to_absolute)
|
|
|
| @parameterized.parameters(
|
| {'batch_size': 1, 'max_gt_boxes': None, 'scale_to_absolute': True},
|
| {'batch_size': 8, 'max_gt_boxes': [1], 'scale_to_absolute': True},
|
| {'batch_size': 1, 'max_gt_boxes': None, 'scale_to_absolute': False},
|
| {'batch_size': 8, 'max_gt_boxes': [1], 'scale_to_absolute': False}
|
| )
|
| @unittest.skipIf(tf_version.is_tf2(), 'Only compatible with TF1.X')
|
| def test_get_eval_metric_ops_for_coco_detections(self, batch_size=1,
|
| max_gt_boxes=None,
|
| scale_to_absolute=False):
|
| eval_config = eval_pb2.EvalConfig()
|
| eval_config.metrics_set.extend(['coco_detection_metrics'])
|
| categories = self._get_categories_list()
|
| eval_dict = self._make_evaluation_dict(batch_size=batch_size,
|
| max_gt_boxes=max_gt_boxes,
|
| scale_to_absolute=scale_to_absolute)
|
| metric_ops = eval_util.get_eval_metric_ops_for_evaluators(
|
| eval_config, categories, eval_dict)
|
| _, update_op = metric_ops['DetectionBoxes_Precision/mAP']
|
|
|
| with self.test_session() as sess:
|
| metrics = {}
|
| for key, (value_op, _) in six.iteritems(metric_ops):
|
| metrics[key] = value_op
|
| sess.run(update_op)
|
| metrics = sess.run(metrics)
|
| self.assertAlmostEqual(1.0, metrics['DetectionBoxes_Precision/mAP'])
|
| self.assertNotIn('DetectionMasks_Precision/mAP', metrics)
|
|
|
| @parameterized.parameters(
|
| {'batch_size': 1, 'max_gt_boxes': None, 'scale_to_absolute': True},
|
| {'batch_size': 8, 'max_gt_boxes': [1], 'scale_to_absolute': True},
|
| {'batch_size': 1, 'max_gt_boxes': None, 'scale_to_absolute': False},
|
| {'batch_size': 8, 'max_gt_boxes': [1], 'scale_to_absolute': False}
|
| )
|
| @unittest.skipIf(tf_version.is_tf2(), 'Only compatible with TF1.X')
|
| def test_get_eval_metric_ops_for_coco_detections_and_masks(
|
| self, batch_size=1, max_gt_boxes=None, scale_to_absolute=False):
|
| eval_config = eval_pb2.EvalConfig()
|
| eval_config.metrics_set.extend(
|
| ['coco_detection_metrics', 'coco_mask_metrics'])
|
| categories = self._get_categories_list()
|
| eval_dict = self._make_evaluation_dict(batch_size=batch_size,
|
| max_gt_boxes=max_gt_boxes,
|
| scale_to_absolute=scale_to_absolute)
|
| metric_ops = eval_util.get_eval_metric_ops_for_evaluators(
|
| eval_config, categories, eval_dict)
|
| _, update_op_boxes = metric_ops['DetectionBoxes_Precision/mAP']
|
| _, update_op_masks = metric_ops['DetectionMasks_Precision/mAP']
|
|
|
| with self.test_session() as sess:
|
| metrics = {}
|
| for key, (value_op, _) in six.iteritems(metric_ops):
|
| metrics[key] = value_op
|
| sess.run(update_op_boxes)
|
| sess.run(update_op_masks)
|
| metrics = sess.run(metrics)
|
| self.assertAlmostEqual(1.0, metrics['DetectionBoxes_Precision/mAP'])
|
| self.assertAlmostEqual(1.0, metrics['DetectionMasks_Precision/mAP'])
|
|
|
| @parameterized.parameters(
|
| {'batch_size': 1, 'max_gt_boxes': None, 'scale_to_absolute': True},
|
| {'batch_size': 8, 'max_gt_boxes': [1], 'scale_to_absolute': True},
|
| {'batch_size': 1, 'max_gt_boxes': None, 'scale_to_absolute': False},
|
| {'batch_size': 8, 'max_gt_boxes': [1], 'scale_to_absolute': False}
|
| )
|
| @unittest.skipIf(tf_version.is_tf2(), 'Only compatible with TF1.X')
|
| def test_get_eval_metric_ops_for_coco_detections_and_resized_masks(
|
| self, batch_size=1, max_gt_boxes=None, scale_to_absolute=False):
|
| eval_config = eval_pb2.EvalConfig()
|
| eval_config.metrics_set.extend(
|
| ['coco_detection_metrics', 'coco_mask_metrics'])
|
| categories = self._get_categories_list()
|
| eval_dict = self._make_evaluation_dict(batch_size=batch_size,
|
| max_gt_boxes=max_gt_boxes,
|
| scale_to_absolute=scale_to_absolute,
|
| resized_groundtruth_masks=True)
|
| metric_ops = eval_util.get_eval_metric_ops_for_evaluators(
|
| eval_config, categories, eval_dict)
|
| _, update_op_boxes = metric_ops['DetectionBoxes_Precision/mAP']
|
| _, update_op_masks = metric_ops['DetectionMasks_Precision/mAP']
|
|
|
| with self.test_session() as sess:
|
| metrics = {}
|
| for key, (value_op, _) in six.iteritems(metric_ops):
|
| metrics[key] = value_op
|
| sess.run(update_op_boxes)
|
| sess.run(update_op_masks)
|
| metrics = sess.run(metrics)
|
| self.assertAlmostEqual(1.0, metrics['DetectionBoxes_Precision/mAP'])
|
| self.assertAlmostEqual(1.0, metrics['DetectionMasks_Precision/mAP'])
|
|
|
| @unittest.skipIf(tf_version.is_tf2(), 'Only compatible with TF1.X')
|
| def test_get_eval_metric_ops_raises_error_with_unsupported_metric(self):
|
| eval_config = eval_pb2.EvalConfig()
|
| eval_config.metrics_set.extend(['unsupported_metric'])
|
| categories = self._get_categories_list()
|
| eval_dict = self._make_evaluation_dict()
|
| with self.assertRaises(ValueError):
|
| eval_util.get_eval_metric_ops_for_evaluators(
|
| eval_config, categories, eval_dict)
|
|
|
| def test_get_eval_metric_ops_for_evaluators(self):
|
| eval_config = eval_pb2.EvalConfig()
|
| eval_config.metrics_set.extend([
|
| 'coco_detection_metrics', 'coco_mask_metrics',
|
| 'precision_at_recall_detection_metrics'
|
| ])
|
| eval_config.include_metrics_per_category = True
|
| eval_config.recall_lower_bound = 0.2
|
| eval_config.recall_upper_bound = 0.6
|
|
|
| evaluator_options = eval_util.evaluator_options_from_eval_config(
|
| eval_config)
|
| self.assertTrue(evaluator_options['coco_detection_metrics']
|
| ['include_metrics_per_category'])
|
| self.assertFalse(evaluator_options['coco_detection_metrics']
|
| ['skip_predictions_for_unlabeled_class'])
|
| self.assertTrue(
|
| evaluator_options['coco_mask_metrics']['include_metrics_per_category'])
|
| self.assertAlmostEqual(
|
| evaluator_options['precision_at_recall_detection_metrics']
|
| ['recall_lower_bound'], eval_config.recall_lower_bound)
|
| self.assertAlmostEqual(
|
| evaluator_options['precision_at_recall_detection_metrics']
|
| ['recall_upper_bound'], eval_config.recall_upper_bound)
|
| self.assertFalse(evaluator_options['precision_at_recall_detection_metrics']
|
| ['skip_predictions_for_unlabeled_class'])
|
|
|
| def test_get_evaluator_with_evaluator_options(self):
|
| eval_config = eval_pb2.EvalConfig()
|
| eval_config.metrics_set.extend(
|
| ['coco_detection_metrics', 'precision_at_recall_detection_metrics'])
|
| eval_config.include_metrics_per_category = True
|
| eval_config.skip_predictions_for_unlabeled_class = True
|
| eval_config.recall_lower_bound = 0.2
|
| eval_config.recall_upper_bound = 0.6
|
| categories = self._get_categories_list()
|
|
|
| evaluator_options = eval_util.evaluator_options_from_eval_config(
|
| eval_config)
|
| evaluator = eval_util.get_evaluators(eval_config, categories,
|
| evaluator_options)
|
|
|
| self.assertTrue(evaluator[0]._include_metrics_per_category)
|
| self.assertTrue(evaluator[0]._skip_predictions_for_unlabeled_class)
|
| self.assertTrue(evaluator[1]._skip_predictions_for_unlabeled_class)
|
| self.assertAlmostEqual(evaluator[1]._recall_lower_bound,
|
| eval_config.recall_lower_bound)
|
| self.assertAlmostEqual(evaluator[1]._recall_upper_bound,
|
| eval_config.recall_upper_bound)
|
|
|
| def test_get_evaluator_with_no_evaluator_options(self):
|
| eval_config = eval_pb2.EvalConfig()
|
| eval_config.metrics_set.extend(
|
| ['coco_detection_metrics', 'precision_at_recall_detection_metrics'])
|
| eval_config.include_metrics_per_category = True
|
| eval_config.recall_lower_bound = 0.2
|
| eval_config.recall_upper_bound = 0.6
|
| categories = self._get_categories_list()
|
|
|
| evaluator = eval_util.get_evaluators(
|
| eval_config, categories, evaluator_options=None)
|
|
|
|
|
|
|
|
|
| self.assertFalse(evaluator[0]._include_metrics_per_category)
|
| self.assertAlmostEqual(evaluator[1]._recall_lower_bound, 0.0)
|
| self.assertAlmostEqual(evaluator[1]._recall_upper_bound, 1.0)
|
|
|
| def test_get_evaluator_with_keypoint_metrics(self):
|
| eval_config = eval_pb2.EvalConfig()
|
| person_keypoints_metric = eval_config.parameterized_metric.add()
|
| person_keypoints_metric.coco_keypoint_metrics.class_label = 'person'
|
| person_keypoints_metric.coco_keypoint_metrics.keypoint_label_to_sigmas[
|
| 'left_eye'] = 0.1
|
| person_keypoints_metric.coco_keypoint_metrics.keypoint_label_to_sigmas[
|
| 'right_eye'] = 0.2
|
| dog_keypoints_metric = eval_config.parameterized_metric.add()
|
| dog_keypoints_metric.coco_keypoint_metrics.class_label = 'dog'
|
| dog_keypoints_metric.coco_keypoint_metrics.keypoint_label_to_sigmas[
|
| 'tail_start'] = 0.3
|
| dog_keypoints_metric.coco_keypoint_metrics.keypoint_label_to_sigmas[
|
| 'mouth'] = 0.4
|
| categories = self._get_categories_list_with_keypoints()
|
|
|
| evaluator = eval_util.get_evaluators(
|
| eval_config, categories, evaluator_options=None)
|
|
|
|
|
| self.assertLen(evaluator, 3)
|
| self.assertFalse(evaluator[0]._include_metrics_per_category)
|
| self.assertEqual(evaluator[1]._category_name, 'person')
|
| self.assertEqual(evaluator[2]._category_name, 'dog')
|
| self.assertAllEqual(evaluator[1]._keypoint_ids, [0, 3])
|
| self.assertAllEqual(evaluator[2]._keypoint_ids, [1, 2])
|
| self.assertAllClose([0.1, 0.2], evaluator[1]._oks_sigmas)
|
| self.assertAllClose([0.3, 0.4], evaluator[2]._oks_sigmas)
|
|
|
| def test_get_evaluator_with_unmatched_label(self):
|
| eval_config = eval_pb2.EvalConfig()
|
| person_keypoints_metric = eval_config.parameterized_metric.add()
|
| person_keypoints_metric.coco_keypoint_metrics.class_label = 'unmatched'
|
| person_keypoints_metric.coco_keypoint_metrics.keypoint_label_to_sigmas[
|
| 'kpt'] = 0.1
|
| categories = self._get_categories_list_with_keypoints()
|
|
|
| evaluator = eval_util.get_evaluators(
|
| eval_config, categories, evaluator_options=None)
|
| self.assertLen(evaluator, 1)
|
| self.assertNotIsInstance(
|
| evaluator[0], coco_evaluation.CocoKeypointEvaluator)
|
|
|
| def test_padded_image_result_dict(self):
|
|
|
| input_data_fields = fields.InputDataFields
|
| detection_fields = fields.DetectionResultFields
|
| key = tf.constant([str(i) for i in range(2)])
|
|
|
| detection_boxes = np.array([[[0., 0., 1., 1.]], [[0.0, 0.0, 0.5, 0.5]]],
|
| dtype=np.float32)
|
| detection_keypoints = np.array([[0.0, 0.0], [0.5, 0.5], [1.0, 1.0]],
|
| dtype=np.float32)
|
| def graph_fn():
|
| detections = {
|
| detection_fields.detection_boxes:
|
| tf.constant(detection_boxes),
|
| detection_fields.detection_scores:
|
| tf.constant([[1.], [1.]]),
|
| detection_fields.detection_classes:
|
| tf.constant([[1], [2]]),
|
| detection_fields.num_detections:
|
| tf.constant([1, 1]),
|
| detection_fields.detection_keypoints:
|
| tf.tile(
|
| tf.reshape(
|
| tf.constant(detection_keypoints), shape=[1, 1, 3, 2]),
|
| multiples=[2, 1, 1, 1])
|
| }
|
|
|
| gt_boxes = detection_boxes
|
| groundtruth = {
|
| input_data_fields.groundtruth_boxes:
|
| tf.constant(gt_boxes),
|
| input_data_fields.groundtruth_classes:
|
| tf.constant([[1.], [1.]]),
|
| input_data_fields.groundtruth_keypoints:
|
| tf.tile(
|
| tf.reshape(
|
| tf.constant(detection_keypoints), shape=[1, 1, 3, 2]),
|
| multiples=[2, 1, 1, 1])
|
| }
|
|
|
| image = tf.zeros((2, 100, 100, 3), dtype=tf.float32)
|
|
|
| true_image_shapes = tf.constant([[100, 100, 3], [50, 100, 3]])
|
| original_image_spatial_shapes = tf.constant([[200, 200], [150, 300]])
|
|
|
| result = eval_util.result_dict_for_batched_example(
|
| image, key, detections, groundtruth,
|
| scale_to_absolute=True,
|
| true_image_shapes=true_image_shapes,
|
| original_image_spatial_shapes=original_image_spatial_shapes,
|
| max_gt_boxes=tf.constant(1))
|
| return (result[input_data_fields.groundtruth_boxes],
|
| result[input_data_fields.groundtruth_keypoints],
|
| result[detection_fields.detection_boxes],
|
| result[detection_fields.detection_keypoints])
|
| (gt_boxes, gt_keypoints, detection_boxes,
|
| detection_keypoints) = self.execute_cpu(graph_fn, [])
|
| self.assertAllEqual(
|
| [[[0., 0., 200., 200.]], [[0.0, 0.0, 150., 150.]]],
|
| gt_boxes)
|
| self.assertAllClose([[[[0., 0.], [100., 100.], [200., 200.]]],
|
| [[[0., 0.], [150., 150.], [300., 300.]]]],
|
| gt_keypoints)
|
|
|
|
|
| self.assertAllEqual(
|
| [[[0., 0., 200., 200.]], [[0.0, 0.0, 75., 150.]]],
|
| detection_boxes)
|
| self.assertAllClose([[[[0., 0.], [100., 100.], [200., 200.]]],
|
| [[[0., 0.], [75., 150.], [150., 300.]]]],
|
| detection_keypoints)
|
|
|
| def test_evaluator_options_from_eval_config_no_super_categories(self):
|
| eval_config_text_proto = """
|
| metrics_set: "coco_detection_metrics"
|
| metrics_set: "coco_mask_metrics"
|
| include_metrics_per_category: true
|
| use_moving_averages: false
|
| batch_size: 1;
|
| """
|
| eval_config = eval_pb2.EvalConfig()
|
| text_format.Merge(eval_config_text_proto, eval_config)
|
| evaluator_options = eval_util.evaluator_options_from_eval_config(
|
| eval_config)
|
| self.assertNotIn('super_categories', evaluator_options['coco_mask_metrics'])
|
|
|
| def test_evaluator_options_from_eval_config_with_super_categories(self):
|
| eval_config_text_proto = """
|
| metrics_set: "coco_detection_metrics"
|
| metrics_set: "coco_mask_metrics"
|
| include_metrics_per_category: true
|
| use_moving_averages: false
|
| batch_size: 1;
|
| super_categories {
|
| key: "supercat1"
|
| value: "a,b,c"
|
| }
|
| super_categories {
|
| key: "supercat2"
|
| value: "d,e,f"
|
| }
|
| """
|
| eval_config = eval_pb2.EvalConfig()
|
| text_format.Merge(eval_config_text_proto, eval_config)
|
| evaluator_options = eval_util.evaluator_options_from_eval_config(
|
| eval_config)
|
| self.assertIn('super_categories', evaluator_options['coco_mask_metrics'])
|
| super_categories = evaluator_options[
|
| 'coco_mask_metrics']['super_categories']
|
| self.assertIn('supercat1', super_categories)
|
| self.assertIn('supercat2', super_categories)
|
| self.assertAllEqual(super_categories['supercat1'], ['a', 'b', 'c'])
|
| self.assertAllEqual(super_categories['supercat2'], ['d', 'e', 'f'])
|
|
|
|
|
| if __name__ == '__main__':
|
| tf.test.main()
|
|
|
