import torch import matplotlib import matplotlib.pyplot as plt from extract_results import extract_results def get_plot_draw_styles(): plot_draw_style = [{'color': (1.0, 0.0, 0.0), 'line_style': '-'}, {'color': (0.0, 1.0, 0.0), 'line_style': '-'}, {'color': (0.0, 0.0, 1.0), 'line_style': '-'}, {'color': (0.0, 0.0, 0.0), 'line_style': '-'}, {'color': (1.0, 0.0, 1.0), 'line_style': '-'}, {'color': (0.0, 1.0, 1.0), 'line_style': '-'}, {'color': (0.5, 0.5, 0.5), 'line_style': '-'}, {'color': (136.0 / 255.0, 0.0, 21.0 / 255.0), 'line_style': '-'}, {'color': (1.0, 127.0 / 255.0, 39.0 / 255.0), 'line_style': '-'}, {'color': (0.0, 162.0 / 255.0, 232.0 / 255.0), 'line_style': '-'}, {'color': (0.0, 0.5, 0.0), 'line_style': '-'}, {'color': (1.0, 0.5, 0.2), 'line_style': '-'}, {'color': (0.1, 0.4, 0.0), 'line_style': '-'}, {'color': (0.6, 0.3, 0.9), 'line_style': '-'}, {'color': (0.4, 0.7, 0.1), 'line_style': '-'}, {'color': (0.2, 0.1, 0.7), 'line_style': '-'}, {'color': (0.7, 0.6, 0.2), 'line_style': '-'}] return plot_draw_style def merge_multiple_runs(eval_data): new_tracker_names = [] ave_success_rate_plot_overlap_merged = [] ave_success_rate_plot_center_merged = [] ave_success_rate_plot_center_norm_merged = [] avg_overlap_all_merged = [] ave_success_rate_plot_overlap = torch.tensor(eval_data['ave_success_rate_plot_overlap']) ave_success_rate_plot_center = torch.tensor(eval_data['ave_success_rate_plot_center']) ave_success_rate_plot_center_norm = torch.tensor(eval_data['ave_success_rate_plot_center_norm']) avg_overlap_all = torch.tensor(eval_data['avg_overlap_all']) trackers = eval_data['trackers'] merged = torch.zeros(len(trackers), dtype=torch.uint8) for i in range(len(trackers)): if merged[i]: continue base_tracker = trackers[i] new_tracker_names.append(base_tracker) match = [True] * len(trackers) match = torch.tensor(match) ave_success_rate_plot_overlap_merged.append(ave_success_rate_plot_overlap[:, match, :].mean(1)) ave_success_rate_plot_center_merged.append(ave_success_rate_plot_center[:, match, :].mean(1)) ave_success_rate_plot_center_norm_merged.append(ave_success_rate_plot_center_norm[:, match, :].mean(1)) avg_overlap_all_merged.append(avg_overlap_all[:, match].mean(1)) merged[match] = 1 ave_success_rate_plot_overlap_merged = torch.stack(ave_success_rate_plot_overlap_merged, dim=1) ave_success_rate_plot_center_merged = torch.stack(ave_success_rate_plot_center_merged, dim=1) ave_success_rate_plot_center_norm_merged = torch.stack(ave_success_rate_plot_center_norm_merged, dim=1) avg_overlap_all_merged = torch.stack(avg_overlap_all_merged, dim=1) eval_data['trackers'] = new_tracker_names eval_data['ave_success_rate_plot_overlap'] = ave_success_rate_plot_overlap_merged.tolist() eval_data['ave_success_rate_plot_center'] = ave_success_rate_plot_center_merged.tolist() eval_data['ave_success_rate_plot_center_norm'] = ave_success_rate_plot_center_norm_merged.tolist() eval_data['avg_overlap_all'] = avg_overlap_all_merged.tolist() return eval_data def plot_draw_save(y, x, scores, trackers, plot_draw_styles, result_plot_path, plot_opts): plt.rcParams['text.usetex'] = False plt.rcParams["font.family"] = "Times New Roman" # Plot settings font_size = plot_opts.get('font_size', 20) font_size_axis = plot_opts.get('font_size_axis', 20) line_width = plot_opts.get('line_width', 2) font_size_legend = plot_opts.get('font_size_legend', 20) plot_type = plot_opts['plot_type'] legend_loc = plot_opts['legend_loc'] xlabel = plot_opts['xlabel'] ylabel = plot_opts['ylabel'] ylabel = "%s" % (ylabel.replace('%', '\%')) xlim = plot_opts['xlim'] ylim = plot_opts['ylim'] title = r"$\bf{%s}$" % (plot_opts['title']) matplotlib.rcParams.update({'font.size': font_size}) matplotlib.rcParams.update({'axes.titlesize': font_size_axis}) matplotlib.rcParams.update({'axes.titleweight': 'black'}) matplotlib.rcParams.update({'axes.labelsize': font_size_axis}) fig, ax = plt.subplots() index_sort = scores.argsort(descending=False) plotted_lines = [] legend_text = [] for id, id_sort in enumerate(index_sort): line = ax.plot(x.tolist(), y[id_sort, :].tolist(), linewidth=line_width, color=plot_draw_styles[index_sort.numel() - id - 1]['color'], linestyle=plot_draw_styles[index_sort.numel() - id - 1]['line_style']) plotted_lines.append(line[0]) tracker = trackers[id_sort] disp_name = tracker['disp_name'] legend_text.append('{} [{:.1f}]'.format(disp_name, scores[id_sort])) ax.legend(plotted_lines[::-1], legend_text[::-1], loc=legend_loc, fancybox=False, edgecolor='black', fontsize=font_size_legend, framealpha=1.0) ax.set(xlabel=xlabel, ylabel=ylabel, xlim=xlim, ylim=ylim, title=title) ax.grid(True, linestyle='-.') fig.tight_layout() if result_plot_path != None: fig.savefig('{}/{}_plot.pdf'.format(result_plot_path, plot_type), dpi=300, format='pdf', transparent=True) print("{} for VastTrack has been saved to {}".format(plot_type, result_plot_path)) plt.draw() def plot_results(results_dir, dataset, report_name, plot_types=('success'), save_result_plot=False, save_result_plot_dir=None, **kwargs): """ Plot results for the given trackers args: trackers - List of trackers to evaluate dataset - List of sequences to evaluate report_name - Name of the folder in env_settings.perm_mat_path where the computed results and plots are saved merge_results - If True, multiple random runs for a non-deterministic trackers are averaged plot_types - List of scores to display. Can contain 'success', 'prec' (precision), and 'norm_prec' (normalized precision) """ plot_draw_styles = get_plot_draw_styles() eval_data = extract_results(results_dir, dataset) tracker_names = eval_data['trackers'] valid_sequence = torch.tensor(eval_data['valid_sequence'], dtype=torch.bool) print( '\nPlotting results over {} / {} sequences'.format(valid_sequence.long().sum().item(), valid_sequence.shape[0])) print('\nNow, Generating plots for {}'.format(report_name)) if save_result_plot == True and save_result_plot_dir != None: result_plot_path = save_result_plot_dir else: result_plot_path = None # ******************************** Success Plot ************************************** if 'success' in plot_types: ave_success_rate_plot_overlap = torch.tensor(eval_data['ave_success_rate_plot_overlap']) # Index out valid sequences auc_curve, auc = get_auc_curve(ave_success_rate_plot_overlap, valid_sequence) threshold_set_overlap = torch.tensor(eval_data['threshold_set_overlap']) success_plot_opts = {'plot_type': 'success', 'legend_loc': 'lower left', 'xlabel': 'Overlap threshold', 'ylabel': 'Overlap Precision', 'xlim': (0, 1.0), 'ylim': (0, 88), 'title': 'Success\ plots\ of\ OPE\ on\ VastTrack_{Tst}'} plot_draw_save(auc_curve, threshold_set_overlap, auc, tracker_names, plot_draw_styles, result_plot_path, success_plot_opts) # ******************************** Precision Plot ************************************** if 'prec' in plot_types: ave_success_rate_plot_center = torch.tensor(eval_data['ave_success_rate_plot_center']) # Index out valid sequences prec_curve, prec_score = get_prec_curve(ave_success_rate_plot_center, valid_sequence) threshold_set_center = torch.tensor(eval_data['threshold_set_center']) precision_plot_opts = {'plot_type': 'precision', 'legend_loc': 'lower right', 'xlabel': 'Location error threshold', 'ylabel': 'Precision', 'xlim': (0, 50), 'ylim': (0, 100), 'title': 'Precision\ plot\ of\ OPE\ on\ VastTrack_{Tst}'} plot_draw_save(prec_curve, threshold_set_center, prec_score, tracker_names, plot_draw_styles, result_plot_path, precision_plot_opts) # ******************************** Norm Precision Plot ************************************** if 'norm_prec' in plot_types: ave_success_rate_plot_center_norm = torch.tensor(eval_data['ave_success_rate_plot_center_norm']) # Index out valid sequences prec_curve, prec_score = get_prec_curve(ave_success_rate_plot_center_norm, valid_sequence) threshold_set_center_norm = torch.tensor(eval_data['threshold_set_center_norm']) norm_precision_plot_opts = {'plot_type': 'norm_precision', 'legend_loc': 'lower right', 'xlabel': 'Location error threshold', 'ylabel': 'Precision', 'xlim': (0, 0.5), 'ylim': (0, 85), 'title': 'Normalized\ Precision\ plots\ of\ OPE\ on\ VastTrack_{Tst}'} plot_draw_save(prec_curve, threshold_set_center_norm, prec_score, tracker_names, plot_draw_styles, result_plot_path, norm_precision_plot_opts) plt.show() def print_results(results_dir, dataset, merge_results=False, plot_types=('success'), **kwargs): """ Print the results for the given trackers in a formatted table args: trackers - List of trackers to evaluate dataset - List of sequences to evaluate report_name - Name of the folder in env_settings.perm_mat_path where the computed results and plots are saved merge_results - If True, multiple random runs for a non-deterministic trackers are averaged plot_types - List of scores to display. Can contain 'success' (prints AUC, OP50, and OP75 scores), 'prec' (prints precision score), and 'norm_prec' (prints normalized precision score) """ eval_data = extract_results(results_dir, dataset) # Merge results from multiple runs if merge_results: eval_data = merge_multiple_runs(eval_data) tracker_names = eval_data['trackers'] valid_sequence = torch.tensor(eval_data['valid_sequence'], dtype=torch.bool) print('\nReporting results over {} / {} sequences'.format(valid_sequence.long().sum().item(), valid_sequence.shape[0])) scores = {} # ******************************** Success Plot ************************************** if 'success' in plot_types: threshold_set_overlap = torch.tensor(eval_data['threshold_set_overlap']) ave_success_rate_plot_overlap = torch.tensor(eval_data['ave_success_rate_plot_overlap']) # Index out valid sequences auc_curve, auc = get_auc_curve(ave_success_rate_plot_overlap, valid_sequence) scores['AUC'] = auc scores['OP50'] = auc_curve[:, threshold_set_overlap == 0.50] scores['OP75'] = auc_curve[:, threshold_set_overlap == 0.75] # ******************************** Precision Plot ************************************** if 'prec' in plot_types: ave_success_rate_plot_center = torch.tensor(eval_data['ave_success_rate_plot_center']) # Index out valid sequences prec_curve, prec_score = get_prec_curve(ave_success_rate_plot_center, valid_sequence) scores['Precision'] = prec_score # ******************************** Norm Precision Plot ********************************* if 'norm_prec' in plot_types: ave_success_rate_plot_center_norm = torch.tensor(eval_data['ave_success_rate_plot_center_norm']) # Index out valid sequences norm_prec_curve, norm_prec_score = get_prec_curve(ave_success_rate_plot_center_norm, valid_sequence) scores['Norm Precision'] = norm_prec_score # Print tracker_disp_names = [trk['disp_name'] for trk in tracker_names] report_text = generate_formatted_report(tracker_disp_names, scores, table_name="VastTrack_results") print(report_text) def get_auc_curve(ave_success_rate_plot_overlap, valid_sequence): ave_success_rate_plot_overlap = ave_success_rate_plot_overlap[valid_sequence, :, :] auc_curve = ave_success_rate_plot_overlap.mean(0) * 100.0 auc = auc_curve.mean(-1) return auc_curve, auc def get_prec_curve(ave_success_rate_plot_center, valid_sequence): ave_success_rate_plot_center = ave_success_rate_plot_center[valid_sequence, :, :] prec_curve = ave_success_rate_plot_center.mean(0) * 100.0 prec_score = prec_curve[:, 20] return prec_curve, prec_score def generate_formatted_report(row_labels, scores, table_name=''): name_width = max([len(d) for d in row_labels] + [len(table_name)]) + 5 min_score_width = 10 report_text = '\n{label: <{width}} |'.format(label=table_name, width=name_width) score_widths = [max(min_score_width, len(k) + 3) for k in scores.keys()] for s, s_w in zip(scores.keys(), score_widths): report_text = '{prev} {s: <{width}} |'.format(prev=report_text, s=s, width=s_w) report_text = '{prev}\n'.format(prev=report_text) for trk_id, d_name in enumerate(row_labels): # display name report_text = '{prev}{tracker: <{width}} |'.format(prev=report_text, tracker=d_name, width=name_width) for (score_type, score_value), s_w in zip(scores.items(), score_widths): report_text = '{prev} {score: <{width}} |'.format(prev=report_text, score='{:0.2f}'.format(score_value[trk_id].item()), width=s_w) report_text = '{prev}\n'.format(prev=report_text) return report_text