|
|
|
|
|
"""This file converts the output proposal file of proposal generator (BSN, BMN)
|
|
|
into the input proposal file of action classifier (Currently supports SSN and
|
|
|
P-GCN, not including TSN, I3D etc.)."""
|
|
|
import argparse
|
|
|
|
|
|
import mmengine
|
|
|
import numpy as np
|
|
|
|
|
|
from mmaction.evaluation import pairwise_temporal_iou
|
|
|
|
|
|
|
|
|
def load_annotations(ann_file):
|
|
|
"""Load the annotation according to ann_file into video_infos."""
|
|
|
video_infos = []
|
|
|
anno_database = mmengine.load(ann_file)
|
|
|
for video_name in anno_database:
|
|
|
video_info = anno_database[video_name]
|
|
|
video_info['video_name'] = video_name
|
|
|
video_infos.append(video_info)
|
|
|
return video_infos
|
|
|
|
|
|
|
|
|
def import_ground_truth(video_infos, activity_index):
|
|
|
"""Read ground truth data from video_infos."""
|
|
|
ground_truth = {}
|
|
|
for video_info in video_infos:
|
|
|
video_id = video_info['video_name'][2:]
|
|
|
this_video_ground_truths = []
|
|
|
for ann in video_info['annotations']:
|
|
|
t_start, t_end = ann['segment']
|
|
|
label = activity_index[ann['label']]
|
|
|
this_video_ground_truths.append([t_start, t_end, label])
|
|
|
ground_truth[video_id] = np.array(this_video_ground_truths)
|
|
|
return ground_truth
|
|
|
|
|
|
|
|
|
def import_proposals(result_dict):
|
|
|
"""Read predictions from result dict."""
|
|
|
proposals = {}
|
|
|
num_proposals = 0
|
|
|
for video_id in result_dict:
|
|
|
result = result_dict[video_id]
|
|
|
this_video_proposals = []
|
|
|
for proposal in result:
|
|
|
t_start, t_end = proposal['segment']
|
|
|
score = proposal['score']
|
|
|
this_video_proposals.append([t_start, t_end, score])
|
|
|
num_proposals += 1
|
|
|
proposals[video_id] = np.array(this_video_proposals)
|
|
|
return proposals, num_proposals
|
|
|
|
|
|
|
|
|
def dump_formatted_proposal(video_idx, video_id, num_frames, fps, gts,
|
|
|
proposals, tiou, t_overlap_self,
|
|
|
formatted_proposal_file):
|
|
|
"""dump the formatted proposal file, which is the input proposal file of
|
|
|
action classifier (e.g: SSN).
|
|
|
|
|
|
Args:
|
|
|
video_idx (int): Index of video.
|
|
|
video_id (str): ID of video.
|
|
|
num_frames (int): Total frames of the video.
|
|
|
fps (float): Fps of the video.
|
|
|
gts (np.ndarray[float]): t_start, t_end and label of groundtruths.
|
|
|
proposals (np.ndarray[float]): t_start, t_end and score of proposals.
|
|
|
tiou (np.ndarray[float]): 2-dim array with IoU ratio.
|
|
|
t_overlap_self (np.ndarray[float]): 2-dim array with overlap_self
|
|
|
(union / self_len) ratio.
|
|
|
formatted_proposal_file (open file object): Open file object of
|
|
|
formatted_proposal_file.
|
|
|
"""
|
|
|
|
|
|
formatted_proposal_file.write(
|
|
|
f'#{video_idx}\n{video_id}\n{num_frames}\n{fps}\n{gts.shape[0]}\n')
|
|
|
for gt in gts:
|
|
|
formatted_proposal_file.write(f'{int(gt[2])} {gt[0]} {gt[1]}\n')
|
|
|
formatted_proposal_file.write(f'{proposals.shape[0]}\n')
|
|
|
|
|
|
best_iou = np.amax(tiou, axis=0)
|
|
|
best_iou_index = np.argmax(tiou, axis=0)
|
|
|
best_overlap = np.amax(t_overlap_self, axis=0)
|
|
|
best_overlap_index = np.argmax(t_overlap_self, axis=0)
|
|
|
|
|
|
for i in range(proposals.shape[0]):
|
|
|
index_iou = best_iou_index[i]
|
|
|
index_overlap = best_overlap_index[i]
|
|
|
label_iou = gts[index_iou][2]
|
|
|
label_overlap = gts[index_overlap][2]
|
|
|
if label_iou != label_overlap:
|
|
|
label = label_iou if label_iou != 0 else label_overlap
|
|
|
else:
|
|
|
label = label_iou
|
|
|
if best_iou[i] == 0 and best_overlap[i] == 0:
|
|
|
formatted_proposal_file.write(
|
|
|
f'0 0 0 {proposals[i][0]} {proposals[i][1]}\n')
|
|
|
else:
|
|
|
formatted_proposal_file.write(
|
|
|
f'{int(label)} {best_iou[i]} {best_overlap[i]} '
|
|
|
f'{proposals[i][0]} {proposals[i][1]}\n')
|
|
|
|
|
|
|
|
|
def parse_args():
|
|
|
parser = argparse.ArgumentParser(description='convert proposal format')
|
|
|
parser.add_argument(
|
|
|
'--ann-file',
|
|
|
type=str,
|
|
|
default='../../../data/ActivityNet/anet_anno_val.json',
|
|
|
help='name of annotation file')
|
|
|
parser.add_argument(
|
|
|
'--activity-index-file',
|
|
|
type=str,
|
|
|
default='../../../data/ActivityNet/anet_activity_indexes_val.txt',
|
|
|
help='name of activity index file')
|
|
|
parser.add_argument(
|
|
|
'--proposal-file',
|
|
|
type=str,
|
|
|
default='../../../results.json',
|
|
|
help='name of proposal file, which is the'
|
|
|
'output of proposal generator (BMN)')
|
|
|
parser.add_argument(
|
|
|
'--formatted-proposal-file',
|
|
|
type=str,
|
|
|
default='../../../anet_val_formatted_proposal.txt',
|
|
|
help='name of formatted proposal file, which is the'
|
|
|
'input of action classifier (SSN)')
|
|
|
args = parser.parse_args()
|
|
|
|
|
|
return args
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
args = parse_args()
|
|
|
formatted_proposal_file = open(args.formatted_proposal_file, 'w')
|
|
|
|
|
|
|
|
|
|
|
|
activity_index, class_idx = {}, 0
|
|
|
for line in open(args.activity_index_file).readlines():
|
|
|
activity_index[line.strip()] = class_idx
|
|
|
class_idx += 1
|
|
|
|
|
|
video_infos = load_annotations(args.ann_file)
|
|
|
ground_truth = import_ground_truth(video_infos, activity_index)
|
|
|
proposal, num_proposals = import_proposals(
|
|
|
mmengine.load(args.proposal_file)['results'])
|
|
|
video_idx = 0
|
|
|
|
|
|
for video_info in video_infos:
|
|
|
video_id = video_info['video_name'][2:]
|
|
|
num_frames = video_info['duration_frame']
|
|
|
fps = video_info['fps']
|
|
|
tiou, t_overlap = pairwise_temporal_iou(
|
|
|
proposal[video_id][:, :2].astype(float),
|
|
|
ground_truth[video_id][:, :2].astype(float),
|
|
|
calculate_overlap_self=True)
|
|
|
|
|
|
dump_formatted_proposal(video_idx, video_id, num_frames, fps,
|
|
|
ground_truth[video_id], proposal[video_id],
|
|
|
tiou, t_overlap, formatted_proposal_file)
|
|
|
video_idx += 1
|
|
|
formatted_proposal_file.close()
|
|
|
|
