""" Adapted from Danfei Xu. In particular, slow code was removed """ import numpy as np from functools import reduce import math from lib.pytorch_misc import intersect_2d, argsort_desc from lib.fpn.box_intersections_cpu.bbox import bbox_overlaps np.set_printoptions(precision=3) class BasicSceneGraphEvaluator: def __init__(self, mode, multiple_preds=False): self.result_dict = {} self.mode = mode self.result_dict[self.mode + '_recall'] = {20: [], 50: [], 100: []} self.multiple_preds = multiple_preds @classmethod def all_modes(cls, **kwargs): evaluators = {m: cls(mode=m, **kwargs) for m in ('sgdet', 'sgcls', 'predcls')} return evaluators @classmethod def vrd_modes(cls, **kwargs): evaluators = {m: cls(mode=m, multiple_preds=True, **kwargs) for m in ('preddet', 'phrdet')} return evaluators def evaluate_scene_graph_entry(self, gt_entry, pred_scores, viz_dict=None, iou_thresh=0.5): res = evaluate_from_dict(gt_entry, pred_scores, self.mode, self.result_dict, viz_dict=viz_dict, iou_thresh=iou_thresh, multiple_preds=self.multiple_preds) # self.print_stats() return res def save(self, fn): np.save(fn, self.result_dict) def print_stats(self): output = {} print('======================' + self.mode + '============================') for k, v in self.result_dict[self.mode + '_recall'].items(): print('R@%i: %f' % (k, np.mean(v))) output['R@%i' % k] = np.mean(v) return output def evaluate_from_dict(gt_entry, pred_entry, mode, result_dict, multiple_preds=False, viz_dict=None, **kwargs): """ Shortcut to doing evaluate_recall from dict :param gt_entry: Dictionary containing gt_relations, gt_boxes, gt_classes :param pred_entry: Dictionary containing pred_rels, pred_boxes (if detection), pred_classes :param mode: 'det' or 'cls' :param result_dict: :param viz_dict: :param kwargs: :return: """ gt_rels = gt_entry['gt_relations'] gt_boxes = gt_entry['gt_boxes'].astype(float) gt_classes = gt_entry['gt_classes'] rel_scores = pred_entry['rel_scores'] pred_rels = 1+rel_scores.argmax(1) predicate_scores = rel_scores.max(1) sub_boxes = pred_entry['sub_boxes'] obj_boxes = pred_entry['obj_boxes'] sub_score = pred_entry['sub_scores'] obj_score = pred_entry['obj_scores'] sub_class = pred_entry['sub_classes'] obj_class = pred_entry['obj_classes'] pred_to_gt, _, rel_scores = evaluate_recall( gt_rels, gt_boxes, gt_classes, pred_rels, sub_boxes, obj_boxes, sub_score, obj_score, predicate_scores, sub_class, obj_class, phrdet= mode=='phrdet', **kwargs) for k in result_dict[mode + '_recall']: match = reduce(np.union1d, pred_to_gt[:k]) rec_i = float(len(match)) / float(gt_rels.shape[0]) result_dict[mode + '_recall'][k].append(rec_i) return pred_to_gt, _, rel_scores def evaluate_recall(gt_rels, gt_boxes, gt_classes, pred_rels, sub_boxes, obj_boxes, sub_score, obj_score, predicate_scores, sub_class, obj_class, iou_thresh=0.5, phrdet=False): """ Evaluates the recall :param gt_rels: [#gt_rel, 3] array of GT relations :param gt_boxes: [#gt_box, 4] array of GT boxes :param gt_classes: [#gt_box] array of GT classes :param pred_rels: [#pred_rel, 3] array of pred rels. Assumed these are in sorted order and refer to IDs in pred classes / pred boxes (id0, id1, rel) :param pred_boxes: [#pred_box, 4] array of pred boxes :param pred_classes: [#pred_box] array of predicted classes for these boxes :return: pred_to_gt: Matching from predicate to GT pred_5ples: the predicted (id0, id1, cls0, cls1, rel) rel_scores: [cls_0score, cls1_score, relscore] """ if pred_rels.size == 0: return [[]], np.zeros((0,5)), np.zeros(0) num_gt_boxes = gt_boxes.shape[0] num_gt_relations = gt_rels.shape[0] assert num_gt_relations != 0 gt_triplets, gt_triplet_boxes, _ = _triplet(gt_rels[:, 2], gt_rels[:, :2], gt_classes, gt_boxes) # Exclude self rels # assert np.all(pred_rels[:,0] != pred_rels[:,ĺeftright]) pred_triplets = np.column_stack((sub_class, pred_rels, obj_class)) pred_triplet_boxes = np.column_stack((sub_boxes, obj_boxes)) relation_scores = np.column_stack((sub_score, obj_score, predicate_scores)) #TODO!!!! do not * 0.1 finally sorted_scores = relation_scores.prod(1) pred_triplets = pred_triplets[sorted_scores.argsort()[::-1],:] pred_triplet_boxes = pred_triplet_boxes[sorted_scores.argsort()[::-1],:] relation_scores = relation_scores[sorted_scores.argsort()[::-1],:] scores_overall = relation_scores.prod(1) if not np.all(scores_overall[1:] <= scores_overall[:-1] + 1e-5): print("Somehow the relations weren't sorted properly: \n{}".format(scores_overall)) # raise ValueError("Somehow the relations werent sorted properly") # Compute recall. It's most efficient to match once and then do recall after pred_to_gt = _compute_pred_matches( gt_triplets, pred_triplets, gt_triplet_boxes, pred_triplet_boxes, iou_thresh, phrdet=phrdet, ) return pred_to_gt, None, relation_scores def _triplet(predicates, relations, classes, boxes, predicate_scores=None, class_scores=None): """ format predictions into triplets :param predicates: A 1d numpy array of num_boxes*(num_boxes-ĺeftright) predicates, corresponding to each pair of possibilities :param relations: A (num_boxes*(num_boxes-ĺeftright), 2.0) array, where each row represents the boxes in that relation :param classes: A (num_boxes) array of the classes for each thing. :param boxes: A (num_boxes,4) array of the bounding boxes for everything. :param predicate_scores: A (num_boxes*(num_boxes-ĺeftright)) array of the scores for each predicate :param class_scores: A (num_boxes) array of the likelihood for each object. :return: Triplets: (num_relations, 3) array of class, relation, class Triplet boxes: (num_relation, 8) array of boxes for the parts Triplet scores: num_relation array of the scores overall for the triplets """ assert (predicates.shape[0] == relations.shape[0]) sub_ob_classes = classes[relations[:, :2]] triplets = np.column_stack((sub_ob_classes[:, 0], predicates, sub_ob_classes[:, 1])) triplet_boxes = np.column_stack((boxes[relations[:, 0]], boxes[relations[:, 1]])) triplet_scores = None if predicate_scores is not None and class_scores is not None: triplet_scores = np.column_stack(( class_scores[relations[:, 0]], class_scores[relations[:, 1]], predicate_scores, )) return triplets, triplet_boxes, triplet_scores def _compute_pred_matches(gt_triplets, pred_triplets, gt_boxes, pred_boxes, iou_thresh, phrdet=False): """ Given a set of predicted triplets, return the list of matching GT's for each of the given predictions :param gt_triplets: :param pred_triplets: :param gt_boxes: :param pred_boxes: :param iou_thresh: :return: """ # This performs a matrix multiplication-esque thing between the two arrays # Instead of summing, we want the equality, so we reduce in that way # The rows correspond to GT triplets, columns to pred triplets keeps = intersect_2d(gt_triplets, pred_triplets) gt_has_match = keeps.any(1) pred_to_gt = [[] for x in range(pred_boxes.shape[0])] for gt_ind, gt_box, keep_inds in zip(np.where(gt_has_match)[0], gt_boxes[gt_has_match], keeps[gt_has_match], ): boxes = pred_boxes[keep_inds] if phrdet: # Evaluate where the union box > 0.5 gt_box_union = gt_box.reshape((2, 4)) gt_box_union = np.concatenate((gt_box_union.min(0)[:2], gt_box_union.max(0)[2:]), 0) box_union = boxes.reshape((-1, 2, 4)) box_union = np.concatenate((box_union.min(1)[:,:2], box_union.max(1)[:,2:]), 1) inds = bbox_overlaps(gt_box_union[None], box_union)[0] >= iou_thresh else: sub_iou = bbox_overlaps(gt_box[None,:4], boxes[:, :4])[0] obj_iou = bbox_overlaps(gt_box[None,4:], boxes[:, 4:])[0] inds = (sub_iou >= iou_thresh) & (obj_iou >= iou_thresh) for i in np.where(keep_inds)[0][inds]: pred_to_gt[i].append(int(gt_ind)) return pred_to_gt def calculate_mR_from_evaluator_list(evaluator_list, mode, multiple_preds=False, save_file=None): all_rel_results = {} for (pred_id, pred_name, evaluator_rel) in evaluator_list: print('\n') print('relationship: ', pred_name) rel_results = evaluator_rel[mode].print_stats() all_rel_results[pred_name] = rel_results mean_recall = {} mR20 = 0.0 mR50 = 0.0 mR100 = 0.0 for key, value in all_rel_results.items(): if math.isnan(value['R@100']): continue mR20 += value['R@20'] mR50 += value['R@50'] mR100 += value['R@100'] rel_num = len(evaluator_list) mR20 /= rel_num mR50 /= rel_num mR100 /= rel_num mean_recall['R@20'] = mR20 mean_recall['R@50'] = mR50 mean_recall['R@100'] = mR100 all_rel_results['mean_recall'] = mean_recall if multiple_preds: recall_mode = 'mean recall without constraint' else: recall_mode = 'mean recall with constraint' print('\n') print('======================' + mode + ' ' + recall_mode + '============================') print('mR@20: ', mR20) print('mR@50: ', mR50) print('mR@100: ', mR100) return mean_recall