File size: 7,158 Bytes
33569f9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 | import numpy as np
import matplotlib.pyplot as plt
from .eval_proposal import ANETproposal
from .eval_detection import ANETdetection
# from .postprocess_utils import multithread_detection , get_infer_dict, load_json
import os
from joblib import Parallel, delayed
import json
import pandas as pd
# def run_evaluation(ground_truth_file, proposal_file, dataset_name='',
# max_avg_nr_proposal=100,
# tiou_thre=np.linspace(0.5, 1.0, 11), subset='test'):
# anet_proposal = ANETproposal(ground_truth_file, proposal_file,
# dataset_name=dataset_name,
# tiou_thresholds=tiou_thre, max_avg_nr_proposals=max_avg_nr_proposal,
# subset=subset, verbose=True, check_status=False)
# anet_proposal.evaluate()
# recall = anet_proposal.recall
# average_recall = anet_proposal.avg_recall
# average_nr_proposal = anet_proposal.proposals_per_video
# return (average_nr_proposal, average_recall, recall)
def load_json(file):
with open(file) as json_file:
data = json.load(json_file)
return data
def plot_metric(args, average_nr_proposals, average_recall, recall, tiou_thresholds=np.linspace(0.5, 1.0, 11)):
fn_size = 14
plt.figure(num=None, figsize=(12, 8))
ax = plt.subplot(1, 1, 1)
colors = ['k', 'r', 'yellow', 'b', 'c', 'm', 'b', 'pink', 'lawngreen', 'indigo']
area_under_curve = np.zeros_like(tiou_thresholds)
for i in range(recall.shape[0]):
area_under_curve[i] = np.trapz(recall[i], average_nr_proposals)
for idx, tiou in enumerate(tiou_thresholds[::2]):
ax.plot(average_nr_proposals, recall[2 * idx, :], color=colors[idx + 1],
label="tiou=[" + str(tiou) + "],area=" + str(int(area_under_curve[2 * idx] * 100) / 100.),
linewidth=4, linestyle='-', marker=None)
ax.plot(average_nr_proposals, average_recall, color=colors[0],
label="tiou=0.5:0.1:1.0," + "area=" + str(int(np.trapz(average_recall, average_nr_proposals) * 100) / 100.),
linewidth=4, linestyle='-', marker=None)
handles, labels = ax.get_legend_handles_labels()
ax.legend([handles[-1]] + handles[:-1], [labels[-1]] + labels[:-1], loc='best')
plt.ylabel('Average Recall', fontsize=fn_size)
plt.xlabel('Average Number of Proposals per Video', fontsize=fn_size)
plt.grid(b=True, which="both")
plt.ylim([0, 1.0])
plt.setp(plt.axes().get_xticklabels(), fontsize=fn_size)
plt.setp(plt.axes().get_yticklabels(), fontsize=fn_size)
plt.savefig(os.path.join(args.output["work_dir"],args.model_name,args.output["output_path"], args.eval["save_fig_path"]))
def evaluation_proposal(gt_filename,pred_filename,tious,subset,max_avg_nr_proposal=100):
anet_proposal = ANETproposal(gt_filename,pred_filename,
tiou_thresholds=tious, max_avg_nr_proposals=max_avg_nr_proposal,
subset=subset, verbose=True, check_status=False)
anet_proposal.evaluate()
recall = anet_proposal.recall
average_recall = anet_proposal.avg_recall
average_nr_proposal = anet_proposal.proposals_per_video
# print("AR@10 is \t", np.mean(recall[:, 9]))
# print("AR@20 is \t", np.mean(recall[:, 19]))
# print("AR@50 is \t", np.mean(recall[:, 49]))
# print("AR@100 is \t", np.mean(recall[:, 99]))
result = f'Proposal: AR@10 {np.mean(recall[:, 9])*100:.3f} \t'
result+=f'AR@20 {np.mean(recall[:, 19])*100:.3f} \t'
result+=f'AR@50 {np.mean(recall[:, 49])*100:.3f} \t'
result+=f'AR@100 {np.mean(recall[:, 99])*100:.3f} \t'
with open(pred_filename.replace('.json','.txt'), 'a') as fobj:
fobj.write(f'{result}\n')
return (np.mean(recall[:, 9])+np.mean(recall[:, 19])+np.mean(recall[:, 49])+np.mean(recall[:, 99]))/4*100
def evaluation_detection(gt_filename,pred_filename,tious,subset):
anet_detection = ANETdetection(
ground_truth_filename=gt_filename,
prediction_filename=pred_filename,
tiou_thresholds=tious,
subset=subset, verbose=True, check_status=False)
anet_detection.evaluate()
mAP_at_tIoU = [f'mAP@{t:.2f} {mAP*100:.3f}' for t, mAP in zip(anet_detection.tiou_thresholds, anet_detection.mAP)]
results = f'Detection: average-mAP {anet_detection.average_mAP*100:.3f} {" ".join(mAP_at_tIoU)}'
# print(results)
with open(pred_filename.replace('.json','.txt'), 'a') as fobj:
fobj.write(f'{results}\n')
return np.mean(anet_detection.mAP)*100
def detection_thread(vid,pred_data,cls_data_cls):
proposal_list = []
old_df = pred_data[pred_data.video_name == vid]
# print(df)
df = pd.DataFrame()
df['score'] = old_df.score.values[:]
df['label'] = old_df.label.values[:]
df['xmin'] = old_df.xmin.values[:]
df['xmax'] = old_df.xmax.values[:]
best_score=np.max(cls_data_cls[vid])
for j in range(min(100, len(df))):
tmp_proposal = {}
tmp_proposal["label"] = 'Fake'
tmp_proposal["score"] = float(df.score.values[j])*best_score
tmp_proposal["segment"] = [max(0, df.xmin.values[j]),
df.xmax.values[j]]
proposal_list.append(tmp_proposal)
return {vid: proposal_list}
def post_process_multi(pred_data,output_file,cls_score_file=None):
pred_videos = list(pred_data.video_name.values[:])
pred_videos = set(pred_videos)
cls_data_cls = {}
if cls_score_file is not None:
best_cls = load_json(cls_score_file)
for idx, vid in enumerate(pred_videos):
if vid in pred_videos:
cls_data_cls[vid] = best_cls[vid]
else:
for idx, vid in enumerate(pred_videos):
if vid in pred_videos:
cls_data_cls[vid] = [1,1]
parallel = Parallel(n_jobs=16, prefer="processes")
detection = parallel(delayed(detection_thread)(vid, pred_data,cls_data_cls)
for vid in pred_videos)
detection_dict = {}
[detection_dict.update(d) for d in detection]
output_dict = {"version": "ANET v1.3, Lavdf", "results": detection_dict, "external_data": {}}
with open(output_file, "w") as out:
json.dump(output_dict, out)
def run_evaluation(preds, ground_truth_file, proposal_file, dataset_name='',
max_avg_nr_proposal=100,
tiou_thre=np.linspace(0.5, 1.0, 11), subset='test',cls_score_file=None):
preds = pd.DataFrame({
'video_name' : preds['video-id'],
'xmin' : preds['t-start'].tolist(),
'xmax': preds['t-end'].tolist(),
'label': preds['label'].tolist(),
'score': preds['score'].tolist()
})
print("saving detection results...")
post_process_multi(preds,proposal_file,cls_score_file)
print("evaluion detection results...")
mAP=evaluation_detection(ground_truth_file,proposal_file,tiou_thre,subset)
print("evaluion proposal results...")
mAR=evaluation_proposal(ground_truth_file,proposal_file,tiou_thre,subset,max_avg_nr_proposal)
return mAP,mAR
|