| | import argparse |
| | import json |
| | import os |
| | import re |
| | import random |
| |
|
| |
|
| | def get_args(): |
| | parser = argparse.ArgumentParser() |
| | parser.add_argument('--base-dir', type=str) |
| | parser.add_argument('--result-file', type=str) |
| | parser.add_argument('--output-file', type=str) |
| | parser.add_argument('--output-result', type=str) |
| | parser.add_argument('--split', type=str, default='test') |
| | parser.add_argument('--options', type=list, default=["A", "B", "C", "D", "E"]) |
| | return parser.parse_args() |
| |
|
| |
|
| | def convert_caps(results): |
| | fakecaps = [] |
| | for result in results: |
| | image_id = result['question_id'] |
| | caption = result['text'] |
| | fakecaps.append({"image_id": int(image_id), "caption": caption}) |
| | return fakecaps |
| |
|
| |
|
| | def get_pred_idx(prediction, choices, options): |
| | """ |
| | Get the index (e.g. 2) from the prediction (e.g. 'C') |
| | """ |
| | if prediction in options[:len(choices)]: |
| | return options.index(prediction) |
| | else: |
| | return -1 |
| | return random.choice(range(len(choices))) |
| |
|
| |
|
| | if __name__ == "__main__": |
| | args = get_args() |
| |
|
| | base_dir = args.base_dir |
| | split_indices = json.load(open(os.path.join(base_dir, "pid_splits.json")))[args.split] |
| | problems = json.load(open(os.path.join(base_dir, "problems.json"))) |
| | predictions = [json.loads(line) for line in open(args.result_file)] |
| | predictions = {pred['question_id']: pred for pred in predictions} |
| | split_problems = {idx: problems[idx] for idx in split_indices} |
| |
|
| | results = {'correct': [], 'incorrect': []} |
| | sqa_results = {} |
| | sqa_results['acc'] = None |
| | sqa_results['correct'] = None |
| | sqa_results['count'] = None |
| | sqa_results['results'] = {} |
| | sqa_results['outputs'] = {} |
| |
|
| | for prob_id, prob in split_problems.items(): |
| | if prob_id not in predictions: |
| | pred = {'text': 'FAILED', 'prompt': 'Unknown'} |
| | pred_text = 'FAILED' |
| | else: |
| | pred = predictions[prob_id] |
| | pred_text = pred['text'] |
| |
|
| | if pred_text in args.options: |
| | answer = pred_text |
| | elif len(pred_text) >= 3 and pred_text[0] in args.options and pred_text[1:3] == ". ": |
| | answer = pred_text[0] |
| | else: |
| | pattern = re.compile(r'The answer is ([A-Z]).') |
| | res = pattern.findall(pred_text) |
| | if len(res) == 1: |
| | answer = res[0] |
| | else: |
| | answer = "FAILED" |
| |
|
| | pred_idx = get_pred_idx(answer, prob['choices'], args.options) |
| |
|
| | analysis = { |
| | 'question_id': prob_id, |
| | 'parsed_ans': answer, |
| | 'ground_truth': args.options[prob['answer']], |
| | 'question': pred['prompt'], |
| | 'pred': pred_text, |
| | 'is_multimodal': '<image>' in pred['prompt'], |
| | } |
| |
|
| | sqa_results['results'][prob_id] = get_pred_idx(answer, prob['choices'], args.options) |
| | sqa_results['outputs'][prob_id] = pred_text |
| |
|
| | if pred_idx == prob['answer']: |
| | results['correct'].append(analysis) |
| | else: |
| | results['incorrect'].append(analysis) |
| |
|
| | correct = len(results['correct']) |
| | total = len(results['correct']) + len(results['incorrect']) |
| |
|
| | |
| | multimodal_correct = len([x for x in results['correct'] if x['is_multimodal']]) |
| | multimodal_incorrect = len([x for x in results['incorrect'] if x['is_multimodal']]) |
| | multimodal_total = multimodal_correct + multimodal_incorrect |
| | |
| |
|
| | print(f'Total: {total}, Correct: {correct}, Accuracy: {correct / total * 100:.2f}%, IMG-Accuracy: {multimodal_correct / multimodal_total * 100:.2f}%') |
| |
|
| | sqa_results['acc'] = correct / total * 100 |
| | sqa_results['correct'] = correct |
| | sqa_results['count'] = total |
| |
|
| | with open(args.output_file, 'w') as f: |
| | json.dump(results, f, indent=2) |
| | with open(args.output_result, 'w') as f: |
| | json.dump(sqa_results, f, indent=2) |
| |
|
