| import os
|
| import json
|
| import argparse
|
|
|
| def eval_pope(answers, label_file):
|
| label_list = [json.loads(q)['label'] for q in open(label_file, 'r')]
|
|
|
| for answer in answers:
|
| text = answer['text']
|
|
|
|
|
| if text.find('.') != -1:
|
| text = text.split('.')[0]
|
|
|
| text = text.replace(',', '')
|
| words = text.split(' ')
|
| if 'No' in words or 'not' in words or 'no' in words:
|
| answer['text'] = 'no'
|
| else:
|
| answer['text'] = 'yes'
|
|
|
| for i in range(len(label_list)):
|
| if label_list[i] == 'no':
|
| label_list[i] = 0
|
| else:
|
| label_list[i] = 1
|
|
|
| pred_list = []
|
| for answer in answers:
|
| if answer['text'] == 'no':
|
| pred_list.append(0)
|
| else:
|
| pred_list.append(1)
|
|
|
| pos = 1
|
| neg = 0
|
| yes_ratio = pred_list.count(1) / len(pred_list)
|
|
|
| TP, TN, FP, FN = 0, 0, 0, 0
|
| for pred, label in zip(pred_list, label_list):
|
| if pred == pos and label == pos:
|
| TP += 1
|
| elif pred == pos and label == neg:
|
| FP += 1
|
| elif pred == neg and label == neg:
|
| TN += 1
|
| elif pred == neg and label == pos:
|
| FN += 1
|
|
|
| print('TP\tFP\tTN\tFN\t')
|
| print('{}\t{}\t{}\t{}'.format(TP, FP, TN, FN))
|
|
|
| precision = float(TP) / float(TP + FP)
|
| recall = float(TP) / float(TP + FN)
|
| f1 = 2*precision*recall / (precision + recall)
|
| acc = (TP + TN) / (TP + TN + FP + FN)
|
| print('Accuracy: {}'.format(acc))
|
| print('Precision: {}'.format(precision))
|
| print('Recall: {}'.format(recall))
|
| print('F1 score: {}'.format(f1))
|
| print('Yes ratio: {}'.format(yes_ratio))
|
| print('%.3f, %.3f, %.3f, %.3f, %.3f' % (f1, acc, precision, recall, yes_ratio) )
|
|
|
| if __name__ == "__main__":
|
| parser = argparse.ArgumentParser()
|
| parser.add_argument("--annotation-dir", type=str)
|
| parser.add_argument("--question-file", type=str)
|
| parser.add_argument("--result-file", type=str)
|
| args = parser.parse_args()
|
|
|
| questions = [json.loads(line) for line in open(args.question_file)]
|
| questions = {question['question_id']: question for question in questions}
|
| answers = [json.loads(q) for q in open(args.result_file)]
|
| for file in os.listdir(args.annotation_dir):
|
| assert file.startswith('coco_pope_')
|
| assert file.endswith('.json')
|
| category = file[10:-5]
|
| cur_answers = [x for x in answers if questions[x['question_id']]['category'] == category]
|
| print('Category: {}, # samples: {}'.format(category, len(cur_answers)))
|
| eval_pope(cur_answers, os.path.join(args.annotation_dir, file))
|
| print("====================================")
|
|
|
