| | import os
|
| | import json
|
| | import argparse
|
| |
|
| | def eval_pope(answers, label_file):
|
| | label_list = [json.loads(q)['label'] for q in open(label_file, 'r')]
|
| |
|
| | for answer in answers:
|
| | text = answer['text']
|
| |
|
| |
|
| | if text.find('.') != -1:
|
| | text = text.split('.')[0]
|
| |
|
| | text = text.replace(',', '')
|
| | words = text.split(' ')
|
| | if 'No' in words or 'not' in words or 'no' in words:
|
| | answer['text'] = 'no'
|
| | else:
|
| | answer['text'] = 'yes'
|
| |
|
| | for i in range(len(label_list)):
|
| | if label_list[i] == 'no':
|
| | label_list[i] = 0
|
| | else:
|
| | label_list[i] = 1
|
| |
|
| | pred_list = []
|
| | for answer in answers:
|
| | if answer['text'] == 'no':
|
| | pred_list.append(0)
|
| | else:
|
| | pred_list.append(1)
|
| |
|
| | pos = 1
|
| | neg = 0
|
| | yes_ratio = pred_list.count(1) / len(pred_list)
|
| |
|
| | TP, TN, FP, FN = 0, 0, 0, 0
|
| | for pred, label in zip(pred_list, label_list):
|
| | if pred == pos and label == pos:
|
| | TP += 1
|
| | elif pred == pos and label == neg:
|
| | FP += 1
|
| | elif pred == neg and label == neg:
|
| | TN += 1
|
| | elif pred == neg and label == pos:
|
| | FN += 1
|
| |
|
| | print('TP\tFP\tTN\tFN\t')
|
| | print('{}\t{}\t{}\t{}'.format(TP, FP, TN, FN))
|
| |
|
| | precision = float(TP) / float(TP + FP)
|
| | recall = float(TP) / float(TP + FN)
|
| | f1 = 2*precision*recall / (precision + recall)
|
| | acc = (TP + TN) / (TP + TN + FP + FN)
|
| | print('Accuracy: {}'.format(acc))
|
| | print('Precision: {}'.format(precision))
|
| | print('Recall: {}'.format(recall))
|
| | print('F1 score: {}'.format(f1))
|
| | print('Yes ratio: {}'.format(yes_ratio))
|
| | print('%.3f, %.3f, %.3f, %.3f, %.3f' % (f1, acc, precision, recall, yes_ratio) )
|
| |
|
| | if __name__ == "__main__":
|
| | parser = argparse.ArgumentParser()
|
| | parser.add_argument("--annotation-dir", type=str)
|
| | parser.add_argument("--question-file", type=str)
|
| | parser.add_argument("--result-file", type=str)
|
| | args = parser.parse_args()
|
| |
|
| | questions = [json.loads(line) for line in open(args.question_file)]
|
| | questions = {question['question_id']: question for question in questions}
|
| | answers = [json.loads(q) for q in open(args.result_file)]
|
| | for file in os.listdir(args.annotation_dir):
|
| | assert file.startswith('coco_pope_')
|
| | assert file.endswith('.json')
|
| | category = file[10:-5]
|
| | cur_answers = [x for x in answers if questions[x['question_id']]['category'] == category]
|
| | print('Category: {}, # samples: {}'.format(category, len(cur_answers)))
|
| | eval_pope(cur_answers, os.path.join(args.annotation_dir, file))
|
| | print("====================================")
|
| |
|
