File size: 6,481 Bytes
f1c1609 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 |
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import argparse
import json
import os
import sys
import torch
import numpy as np
import time
from os.path import dirname, abspath
pdvc_dir = dirname(abspath(__file__))
sys.path.insert(0, pdvc_dir)
sys.path.insert(0, os.path.join(pdvc_dir, 'densevid_eval3'))
sys.path.insert(0, os.path.join(pdvc_dir, 'densevid_eval3/SODA'))
# print(sys.path)
from eval_utils import evaluate
from pdvc.pdvc import build
from misc.utils import create_logger
from data.video_dataset import PropSeqDataset, collate_fn
from torch.utils.data import DataLoader
from os.path import basename
import pandas as pd
def create_fake_test_caption_file(metadata_csv_path):
out = {}
df = pd.read_csv(metadata_csv_path)
for i, row in df.iterrows():
out[basename(row['filename']).split('.')[0]] = {'duration': row['video-duration'], "timestamps": [[0, 0.5]], "sentences":["None"]}
fake_test_json = '.fake_test_json.tmp'
json.dump(out, open(fake_test_json, 'w'))
return fake_test_json
def main(opt):
folder_path = os.path.join(opt.eval_save_dir, opt.eval_folder)
if opt.eval_mode == 'test':
if not os.path.exists(folder_path):
os.makedirs(folder_path)
logger = create_logger(folder_path, 'val.log')
if opt.eval_model_path:
model_path = opt.eval_model_path
infos_path = os.path.join('/'.join(opt.eval_model_path.split('/')[:-1]), 'info.json')
else:
model_path = os.path.join(folder_path, 'model-best.pth')
infos_path = os.path.join(folder_path, 'info.json')
logger.info(vars(opt))
with open(infos_path, 'rb') as f:
logger.info('load info from {}'.format(infos_path))
old_opt = json.load(f)['best']['opt']
for k, v in old_opt.items():
if k[:4] != 'eval':
vars(opt).update({k: v})
opt.transformer_input_type = opt.eval_transformer_input_type
if not torch.cuda.is_available():
opt.nthreads = 0
# Create the Data Loader instance
if opt.eval_mode == 'test':
opt.eval_caption_file = create_fake_test_caption_file(opt.test_video_meta_data_csv_path)
opt.visual_feature_folder = opt.test_video_feature_folder
val_dataset = PropSeqDataset(opt.eval_caption_file,
opt.visual_feature_folder, opt.text_feature_folder,
opt.dict_file, False, opt.eval_proposal_type,
opt)
loader = DataLoader(val_dataset, batch_size=opt.batch_size_for_eval,
shuffle=False, num_workers=opt.nthreads, collate_fn=collate_fn)
model, criterion, contrastive_criterion, postprocessors = build(opt)
model.translator = val_dataset.translator
while not os.path.exists(model_path):
raise AssertionError('File {} does not exist'.format(model_path))
logger.debug('Loading model from {}'.format(model_path))
loaded_pth = torch.load(model_path, map_location=opt.eval_device)
epoch = loaded_pth['epoch']
# loaded_pth = transfer(model, loaded_pth, model_path+'.transfer.pth')
model.load_state_dict(loaded_pth['model'], strict=True)
model.eval()
model.to(opt.eval_device)
if opt.eval_mode == 'test':
out_json_path = os.path.join(folder_path, 'dvc_results.json')
evaluate(model, criterion, postprocessors, loader, out_json_path,
logger, args=opt, alpha=opt.ec_alpha, dvc_eval_version=opt.eval_tool_version, device=opt.eval_device, debug=False, skip_lang_eval=True)
else:
out_json_path = os.path.join(folder_path, '{}_epoch{}_num{}_alpha{}.json'.format(
time.strftime("%Y-%m-%d-%H-%M-%S_", time.localtime()) + str(opt.id), epoch, len(loader.dataset),
opt.ec_alpha))
caption_scores, eval_loss = evaluate(model, criterion, postprocessors, loader, out_json_path,
logger, args=opt, alpha=opt.ec_alpha, dvc_eval_version=opt.eval_tool_version, device=opt.eval_device, debug=False, skip_lang_eval=False)
# breakpoint()
avg_eval_score = {key: np.array(value).mean() for key, value in caption_scores.items() if key !='tiou'}
# avg_eval_score2 = {key: np.array(value).mean() * 4917 / len(loader.dataset) for key, value in caption_scores.items() if key != 'tiou'}
# logger.info(
# '\nValidation result based on all 4917 val videos:\n {}\n avg_score:\n{}'.format(
# caption_scores.items(),
# avg_eval_score))
logger.info(
'\nValidation result based on {} available val videos:\n avg_score:\n{}'.format(len(loader.dataset),
avg_eval_score))
logger.info('saving reults json to {}'.format(out_json_path))
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--eval_save_dir', type=str, default='save')
parser.add_argument('--eval_mode', type=str, default='eval', choices=['eval', 'test'])
parser.add_argument('--test_video_feature_folder', type=str, nargs='+', default=None)
parser.add_argument('--test_video_meta_data_csv_path', type=str, default=None)
parser.add_argument('--eval_folder', type=str, required=True)
parser.add_argument('--eval_model_path', type=str, default='')
parser.add_argument('--eval_tool_version', type=str, default='2018', choices=['2018', '2021'])
parser.add_argument('--eval_caption_file', type=str, default='data/anet/captiondata/val_1.json')
parser.add_argument('--eval_proposal_type', type=str, default='gt')
parser.add_argument('--eval_transformer_input_type', type=str, default='queries', choices=['gt_proposals', 'prior_proposals','queries'])
parser.add_argument('--gpu_id', type=str, nargs='+', default=['0'])
parser.add_argument('--eval_device', type=str, default='cuda')
parser.add_argument('--prior_manner', type=str, default='all', choices=['add', 'all'])
opt = parser.parse_args()
#breakpoint()
os.environ["CUDA_VISIBLE_DEVICES"] = ",".join([str(i) for i in opt.gpu_id])
os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
if True:
torch.backends.cudnn.enabled = False
main(opt)
|