| |
| |
| |
|
|
| import os |
| from os.path import join as pjoin |
| import json |
| import argparse |
|
|
| SPLIT = '613' |
| |
|
|
| RESULTS_ROOT = f'./results/baselines/{SPLIT}' |
| GTS_ROOT = './gts' |
| EVAL_ROOT = f'./eval_results/baselines/{SPLIT}' |
|
|
| TASKS = ['interactable', 'semantics', 'interaction'] |
| IS_FINETUNE = ['', '_finetune'] |
| FORMATS = ['det', 'seg'] |
| METHODS = ['CenterNet2', 'FasterRCNN', 'MaskRCNN', 'YOLO', 'UIED', 'Xianyu', |
| 'GPT4V-E2E', 'Gemini-E2E', 'OmniParser', 'Claude4_5-sonnet-E2E', 'Gemini-2_5-pro-E2E', 'GPT5-E2E', 'O4-E2E','Qwen3-VL-plus-E2E', 'internVL-E2E', 'Seed-E2E', 'CogVLM'] |
| LLM_METHODS = ['GPT4V-E2E', 'Gemini-E2E', 'OmniParser', 'Claude4_5-sonnet-E2E', 'Gemini-2_5-pro-E2E', 'GPT5-E2E', 'O4-E2E', 'Qwen3-VL-plus-E2E', 'internVL-E2E', 'Seed-E2E', 'CogVLM'] |
|
|
| EVAL_DIMENTION = ['i', 's'] |
| GT_JSON = f'{SPLIT}.json' |
|
|
| evaluator_script = './evaluate_coco.py' |
| interactable_mask_evaluator_script = './evaluate_interactable_mask.py' |
|
|
|
|
| def main(args): |
| for format in FORMATS: |
| for task in TASKS: |
| gt_path = pjoin(GTS_ROOT, format, task, GT_JSON) |
| for method in METHODS: |
| for is_finetune in IS_FINETUNE: |
| result_path = pjoin(RESULTS_ROOT, format, task, method + is_finetune + '.json') |
| if not os.path.exists(result_path): |
| continue |
| for eval_dimension in EVAL_DIMENTION: |
| if task == 'interactable' and not eval_dimension == 'i': |
| continue |
| if task == 'semantics' and method in LLM_METHODS and eval_dimension == 'i': |
| continue |
| eval_result_dir = pjoin(EVAL_ROOT, format, task) |
| os.makedirs(eval_result_dir, exist_ok=True) |
| eval_result_path = pjoin(eval_result_dir, method + is_finetune + f'_{eval_dimension}' + '.csv') |
| eval_summary_path = pjoin(eval_result_dir, method + is_finetune + f'_{eval_dimension}' + '_summary.txt') |
|
|
| if os.path.exists(eval_result_path): |
| print(f'{eval_result_path} exists, skipping...') |
| continue |
|
|
| with open(result_path, 'r') as f: |
| result = json.load(f) |
| cli = f'python {evaluator_script} ' + \ |
| f'-d {eval_dimension} ' + \ |
| f'-gt {gt_path} ' + \ |
| f'-dt {result_path} ' + \ |
| f'-i {"bbox" if format == "det" else "segm"} ' + \ |
| f'-l {eval_result_path} ' + \ |
| '-s ' + \ |
| ('-n ' if not (method in LLM_METHODS and task == 'semantics') else '') + \ |
| f'> {eval_summary_path}' |
| print(cli) |
| os.system(cli) |
| if format == 'seg' and task == 'interactable': |
| eval_result_path = pjoin(eval_result_dir, method + is_finetune + f'_mask' + '.txt') |
| cli = f'python {interactable_mask_evaluator_script} ' + \ |
| f'-gt {gt_path} ' + \ |
| f'-dt {result_path} ' + \ |
| f'-l {eval_result_path} ' + \ |
| '-n ' |
| print(cli) |
| os.system(cli) |
|
|
|
|
| if __name__ == '__main__': |
| parser = argparse.ArgumentParser() |
|
|
| args = parser.parse_args() |
| main(args) |
