|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
import warnings |
|
|
|
|
|
warnings.filterwarnings("ignore", category=RuntimeWarning) |
|
|
import os.path as osp |
|
|
import time |
|
|
import argparse |
|
|
import json |
|
|
import numpy as np |
|
|
import multiprocessing as mp |
|
|
import pandas as pd |
|
|
from pycocotools import mask as cocomask |
|
|
|
|
|
from third_parts.revos.utils.metircs import db_eval_iou, db_eval_boundary, get_r2vos_accuracy, get_r2vos_robustness |
|
|
|
|
|
NUM_WOEKERS = 128 |
|
|
|
|
|
def eval_queue(q, rank, out_dict): |
|
|
while not q.empty(): |
|
|
vid_name, exp = q.get() |
|
|
vid = exp_dict[vid_name] |
|
|
exp_name = f'{vid_name}_{exp}' |
|
|
|
|
|
pred = mask_pred_dict[vid_name][exp] |
|
|
|
|
|
vid_len, h, w = len(vid['frames']), vid['height'], vid['width'] |
|
|
gt_masks = np.zeros((vid_len, h, w), dtype=np.uint8) |
|
|
pred_masks = np.zeros((vid_len, h, w), dtype=np.uint8) |
|
|
|
|
|
anno_ids = vid['expressions'][exp]['anno_id'] |
|
|
|
|
|
for frame_idx, frame_name in enumerate(vid['frames']): |
|
|
|
|
|
for anno_id in anno_ids: |
|
|
mask_rle = mask_dict[str(anno_id)][frame_idx] |
|
|
if mask_rle: |
|
|
gt_masks[frame_idx] += cocomask.decode(mask_rle) |
|
|
|
|
|
|
|
|
pred_mask = cocomask.decode(pred['prediction_masks'][frame_idx]) |
|
|
pred_masks[frame_idx] += pred_mask |
|
|
j = db_eval_iou(gt_masks, pred_masks).mean() |
|
|
f = db_eval_boundary(gt_masks, pred_masks).mean() |
|
|
a = get_r2vos_accuracy(gt_masks, pred_masks).mean() |
|
|
|
|
|
out_dict[exp_name] = [j, f, a] |
|
|
|
|
|
|
|
|
if __name__ == '__main__': |
|
|
parser = argparse.ArgumentParser() |
|
|
parser.add_argument("pred_path", type=str, ) |
|
|
parser.add_argument("--exp_path", type=str, default="data/video_datas/revos/meta_expressions_valid_.json") |
|
|
parser.add_argument("--mask_path", type=str, default="data/video_datas/revos/mask_dict.json") |
|
|
parser.add_argument("--save_json_name", type=str, default="revos_valid.json") |
|
|
parser.add_argument("--save_csv_name", type=str, default="revos_valid.csv") |
|
|
args = parser.parse_args() |
|
|
queue = mp.Queue() |
|
|
exp_dict = json.load(open(args.exp_path))['videos'] |
|
|
mask_dict = json.load(open(args.mask_path)) |
|
|
|
|
|
mask_pred = json.load(open(args.pred_path)) |
|
|
|
|
|
shared_exp_dict = mp.Manager().dict(exp_dict) |
|
|
shared_mask_dict = mp.Manager().dict(mask_dict) |
|
|
output_dict = mp.Manager().dict() |
|
|
|
|
|
mask_pred_dict = mp.Manager().dict(mask_pred) |
|
|
|
|
|
|
|
|
for idx, vid_name in enumerate(exp_dict): |
|
|
vid = exp_dict[vid_name] |
|
|
for exp in vid['expressions']: |
|
|
queue.put([vid_name, exp]) |
|
|
|
|
|
start_time = time.time() |
|
|
if NUM_WOEKERS > 1: |
|
|
processes = [] |
|
|
for rank in range(NUM_WOEKERS): |
|
|
p = mp.Process(target=eval_queue, args=(queue, rank, output_dict)) |
|
|
p.start() |
|
|
processes.append(p) |
|
|
|
|
|
for p in processes: |
|
|
p.join() |
|
|
else: |
|
|
eval_queue(queue, 0, output_dict) |
|
|
|
|
|
|
|
|
output_json_path = osp.join(osp.dirname(args.pred_path), args.save_json_name) |
|
|
output_csv_path = osp.join(osp.dirname(args.pred_path), args.save_csv_name) |
|
|
|
|
|
data_list = [] |
|
|
for videxp, (j, f, a) in output_dict.items(): |
|
|
vid_name, exp = videxp.rsplit('_', 1) |
|
|
data = {} |
|
|
|
|
|
data['video_name'] = vid_name |
|
|
data['exp_id'] = exp |
|
|
data['exp'] = exp_dict[vid_name]['expressions'][exp]['exp'] |
|
|
data['videxp'] = videxp |
|
|
data['J'] = round(100 * j, 2) |
|
|
data['F'] = round(100 * f, 2) |
|
|
data['JF'] = round(100 * (j + f) / 2, 2) |
|
|
data['A'] = round(100 * a, 2) |
|
|
data['type_id'] = exp_dict[vid_name]['expressions'][exp]['type_id'] |
|
|
|
|
|
data_list.append(data) |
|
|
|
|
|
is_long = lambda x: x['type_id'] == 0 |
|
|
is_short = lambda x: x['type_id'] == 1 |
|
|
|
|
|
j_referring = np.array([d['J'] for d in data_list if is_long(d)]).mean() |
|
|
f_referring = np.array([d['F'] for d in data_list if is_long(d)]).mean() |
|
|
a_referring = np.array([d['A'] for d in data_list if is_long(d)]).mean() |
|
|
jf_referring = (j_referring + f_referring) / 2 |
|
|
|
|
|
j_reason = np.array([d['J'] for d in data_list if is_short(d)]).mean() |
|
|
f_reason = np.array([d['F'] for d in data_list if is_short(d)]).mean() |
|
|
a_reason = np.array([d['A'] for d in data_list if is_short(d)]).mean() |
|
|
jf_reason = (j_reason + f_reason) / 2 |
|
|
|
|
|
j_referring_reason = (j_referring + j_reason) / 2 |
|
|
f_referring_reason = (f_referring + f_reason) / 2 |
|
|
a_referring_reason = (a_referring + a_reason) / 2 |
|
|
jf_referring_reason = (jf_referring + jf_reason) / 2 |
|
|
|
|
|
results = { |
|
|
"long": { |
|
|
"J" : j_referring, |
|
|
"F" : f_referring, |
|
|
"A" : a_referring, |
|
|
"JF": jf_referring |
|
|
}, |
|
|
"short": { |
|
|
"J" : j_reason, |
|
|
"F" : f_reason, |
|
|
"A" : a_reason, |
|
|
"JF": jf_reason |
|
|
}, |
|
|
"overall": { |
|
|
"J" : j_referring_reason, |
|
|
"F" : f_referring_reason, |
|
|
"A" : a_referring_reason, |
|
|
"JF": jf_referring_reason |
|
|
} |
|
|
} |
|
|
|
|
|
print(results) |
|
|
with open(output_json_path, 'w') as f: |
|
|
json.dump(results, f, indent=4) |
|
|
print(f"Results saved to {output_json_path}") |
|
|
|
|
|
data4csv = {} |
|
|
for data in data_list: |
|
|
for k, v in data.items(): |
|
|
data4csv[k] = data4csv.get(k, []) + [v] |
|
|
|
|
|
df = pd.DataFrame(data4csv) |
|
|
df.to_csv(output_csv_path, index=False) |
|
|
print(f"Results saved to {output_csv_path}") |
|
|
|
|
|
end_time = time.time() |
|
|
total_time = end_time - start_time |
|
|
print("time: %.4f s" %(total_time)) |
|
|
|
