# Multi-HMR # Copyright (c) 2024-present NAVER Corp. # CC BY-NC-SA 4.0 license import torch import numpy as np from itertools import product def compute_prf1(count, miss, fp): """ Code modified from https://github.com/Arthur151/ROMP/blob/4eebd3647f57d291d26423e51f0d514ff7197cb3/simple_romp/evaluation/RH_evaluation/evaluation.py#L90 """ if count == 0: return 0, 0, 0 all_tp = count - miss all_fp = fp all_fn = miss if all_tp == 0: return 0.0, 0.0, 0.0 all_f1_score = round(all_tp / (all_tp + 0.5 * (all_fp + all_fn)), 2) all_recall = round(all_tp / (all_tp + all_fn), 2) all_precision = round(all_tp / (all_tp + all_fp), 2) return 100.0 * all_precision, 100.0 * all_recall, 100.0 * all_f1_score def match_2d_greedy( pred_kps, gtkp, valid_mask, imgPath=None, baseline=None, iou_thresh=0.05, valid=None, ind=-1, ): """ Code modified from: https://github.com/Arthur151/ROMP/blob/4eebd3647f57d291d26423e51f0d514ff7197cb3/simple_romp/trace2/evaluation/eval_3DPW.py#L232 matches groundtruth keypoints to the detection by considering all possible matchings. :return: best possible matching, a list of tuples, where each tuple corresponds to one match of pred_person.to gt_person. the order within one tuple is as follows (idx_pred_kps, idx_gt_kps) """ predList = np.arange(len(pred_kps)) gtList = np.arange(len(gtkp)) # get all pairs of elements in pred_kps, gtkp # all combinations of 2 elements from l1 and l2 combs = list(product(predList, gtList)) errors_per_pair = {} errors_per_pair_list = [] for comb in combs: vmask = valid_mask[comb[1]] assert vmask.sum() > 0, print("no valid points") errors_per_pair[str(comb)] = np.linalg.norm( pred_kps[comb[0]][vmask, :2] - gtkp[comb[1]][vmask, :2], 2 ) errors_per_pair_list.append(errors_per_pair[str(comb)]) gtAssigned = np.zeros((len(gtkp),), dtype=bool) opAssigned = np.zeros((len(pred_kps),), dtype=bool) errors_per_pair_list = np.array(errors_per_pair_list) bestMatch = [] excludedGtBecauseInvalid = [] falsePositiveCounter = 0 while np.sum(gtAssigned) < len(gtAssigned) and np.sum( opAssigned ) + falsePositiveCounter < len(pred_kps): found = False falsePositive = False while not (found): if sum(np.inf == errors_per_pair_list) == len(errors_per_pair_list): print("something went wrong here") minIdx = np.argmin(errors_per_pair_list) minComb = combs[minIdx] # compute IOU iou = get_bbx_overlap( pred_kps[minComb[0]], gtkp[minComb[1]] ) # , imgPath, baseline) # if neither prediction nor ground truth has been matched before and iou # is larger than threshold if ( not (opAssigned[minComb[0]]) and not (gtAssigned[minComb[1]]) and iou >= iou_thresh ): # print(imgPath + ': found matching') found = True errors_per_pair_list[minIdx] = np.inf else: errors_per_pair_list[minIdx] = np.inf # if errors_per_pair_list[minIdx] > # matching_threshold*headBboxs[combs[minIdx][1]]: if iou < iou_thresh: # print( # imgPath + ': false positive detected using threshold') found = True falsePositive = True falsePositiveCounter += 1 # if ground truth of combination is valid keep the match, else exclude # gt from matching if not (valid is None): if valid[minComb[1]]: if not falsePositive: bestMatch.append(minComb) opAssigned[minComb[0]] = True gtAssigned[minComb[1]] = True else: gtAssigned[minComb[1]] = True excludedGtBecauseInvalid.append(minComb[1]) elif not falsePositive: # same as above but without checking for valid bestMatch.append(minComb) opAssigned[minComb[0]] = True gtAssigned[minComb[1]] = True bestMatch = np.array(bestMatch) # add false positives and false negatives to the matching # find which elements have been successfully assigned opAssigned = [] gtAssigned = [] for pair in bestMatch: opAssigned.append(pair[0]) gtAssigned.append(pair[1]) opAssigned.sort() gtAssigned.sort() falsePositives = [] misses = [] # handle false positives opIds = np.arange(len(pred_kps)) # returns values of oIds that are not in opAssigned notAssignedIds = np.setdiff1d(opIds, opAssigned) for notAssignedId in notAssignedIds: falsePositives.append(notAssignedId) gtIds = np.arange(len(gtList)) # returns values of gtIds that are not in gtAssigned notAssignedIdsGt = np.setdiff1d(gtIds, gtAssigned) # handle false negatives/misses for notAssignedIdGt in notAssignedIdsGt: if not (valid is None): # if using the new matching if valid[notAssignedIdGt]: # print(imgPath + ': miss') misses.append(notAssignedIdGt) else: excludedGtBecauseInvalid.append(notAssignedIdGt) else: # print(imgPath + ': miss') misses.append(notAssignedIdGt) return bestMatch, falsePositives, misses # tuples are (idx_pred_kps, idx_gt_kps) def get_bbx_overlap(p1, p2): """ Code modifed from https://github.com/Arthur151/ROMP/blob/4eebd3647f57d291d26423e51f0d514ff7197cb3/simple_romp/trace2/evaluation/eval_3DPW.py#L185 """ min_p1 = np.min(p1, axis=0) min_p2 = np.min(p2, axis=0) max_p1 = np.max(p1, axis=0) max_p2 = np.max(p2, axis=0) bb1 = {} bb2 = {} bb1["x1"] = min_p1[0] bb1["x2"] = max_p1[0] bb1["y1"] = min_p1[1] bb1["y2"] = max_p1[1] bb2["x1"] = min_p2[0] bb2["x2"] = max_p2[0] bb2["y1"] = min_p2[1] bb2["y2"] = max_p2[1] assert bb1["x1"] < bb1["x2"] assert bb1["y1"] < bb1["y2"] assert bb2["x1"] < bb2["x2"] assert bb2["y1"] < bb2["y2"] # determine the coordinates of the intersection rectangle x_left = max(bb1["x1"], bb2["x1"]) y_top = max(bb1["y1"], bb2["y1"]) x_right = min(bb1["x2"], bb2["x2"]) y_bottom = min(bb1["y2"], bb2["y2"]) # The intersection of two axis-aligned bounding boxes is always an # axis-aligned bounding box intersection_area = max(0, x_right - x_left + 1) * max(0, y_bottom - y_top + 1) # compute the area of both AABBs bb1_area = (bb1["x2"] - bb1["x1"] + 1) * (bb1["y2"] - bb1["y1"] + 1) bb2_area = (bb2["x2"] - bb2["x1"] + 1) * (bb2["y2"] - bb2["y1"] + 1) # compute the intersection over union by taking the intersection # area and dividing it by the sum of prediction + ground-truth # areas - the interesection area iou = intersection_area / float(bb1_area + bb2_area - intersection_area) return iou class AverageMeter(object): """ Code mofied from https://github.com/pytorch/examples/blob/main/imagenet/main.py#L423 Computes and stores the average and current value """ def __init__(self, name, fmt=":f"): self.name = name self.fmt = fmt self.reset() def reset(self): self.val = 0 self.avg = 0 self.sum = 0 self.count = 0 def update(self, val, n=1): if type(val) == torch.Tensor: val = val.detach() self.val = val self.sum += val * n self.count += n self.avg = self.sum / self.count def __str__(self): fmtstr = "{name} {val" + self.fmt + "} ({avg" + self.fmt + "})" return fmtstr.format(**self.__dict__)