File size: 3,684 Bytes
6d35aff | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 | # Description: Evaluate baselines on the dataset
#
# Usage: python eval_baselines.py
import os
from os.path import join as pjoin
import json
import argparse
SPLIT = '613'
# SPLIT = 'genre'
RESULTS_ROOT = f'./results/baselines/{SPLIT}'
GTS_ROOT = './gts'
EVAL_ROOT = f'./eval_results/baselines/{SPLIT}'
TASKS = ['interactable', 'semantics', 'interaction']
IS_FINETUNE = ['', '_finetune']
FORMATS = ['det', 'seg']
METHODS = ['CenterNet2', 'FasterRCNN', 'MaskRCNN', 'YOLO', 'UIED', 'Xianyu',
'GPT4V-E2E', 'Gemini-E2E', 'OmniParser', 'Claude4_5-sonnet-E2E', 'Gemini-2_5-pro-E2E', 'GPT5-E2E', 'O4-E2E','Qwen3-VL-plus-E2E', 'internVL-E2E', 'Seed-E2E', 'CogVLM']
LLM_METHODS = ['GPT4V-E2E', 'Gemini-E2E', 'OmniParser', 'Claude4_5-sonnet-E2E', 'Gemini-2_5-pro-E2E', 'GPT5-E2E', 'O4-E2E', 'Qwen3-VL-plus-E2E', 'internVL-E2E', 'Seed-E2E', 'CogVLM']
EVAL_DIMENTION = ['i', 's']
GT_JSON = f'{SPLIT}.json'
evaluator_script = './evaluate_coco.py'
interactable_mask_evaluator_script = './evaluate_interactable_mask.py'
def main(args):
for format in FORMATS:
for task in TASKS:
gt_path = pjoin(GTS_ROOT, format, task, GT_JSON)
for method in METHODS:
for is_finetune in IS_FINETUNE:
result_path = pjoin(RESULTS_ROOT, format, task, method + is_finetune + '.json')
if not os.path.exists(result_path):
continue
for eval_dimension in EVAL_DIMENTION:
if task == 'interactable' and not eval_dimension == 'i':
continue
if task == 'semantics' and method in LLM_METHODS and eval_dimension == 'i':
continue
eval_result_dir = pjoin(EVAL_ROOT, format, task)
os.makedirs(eval_result_dir, exist_ok=True)
eval_result_path = pjoin(eval_result_dir, method + is_finetune + f'_{eval_dimension}' + '.csv')
eval_summary_path = pjoin(eval_result_dir, method + is_finetune + f'_{eval_dimension}' + '_summary.txt')
if os.path.exists(eval_result_path):
print(f'{eval_result_path} exists, skipping...')
continue
with open(result_path, 'r') as f:
result = json.load(f)
cli = f'python {evaluator_script} ' + \
f'-d {eval_dimension} ' + \
f'-gt {gt_path} ' + \
f'-dt {result_path} ' + \
f'-i {"bbox" if format == "det" else "segm"} ' + \
f'-l {eval_result_path} ' + \
'-s ' + \
('-n ' if not (method in LLM_METHODS and task == 'semantics') else '') + \
f'> {eval_summary_path}'
print(cli)
os.system(cli)
if format == 'seg' and task == 'interactable':
eval_result_path = pjoin(eval_result_dir, method + is_finetune + f'_mask' + '.txt')
cli = f'python {interactable_mask_evaluator_script} ' + \
f'-gt {gt_path} ' + \
f'-dt {result_path} ' + \
f'-l {eval_result_path} ' + \
'-n '
print(cli)
os.system(cli)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
args = parser.parse_args()
main(args) |