# Description: Evaluate baselines on the dataset
#
# Usage: python eval_baselines.py

import os
from os.path import join as pjoin
import json
import argparse

SPLIT = '613'
# SPLIT = 'genre'

RESULTS_ROOT = f'./results/baselines/{SPLIT}'
GTS_ROOT = './gts'
EVAL_ROOT = f'./eval_results/baselines/{SPLIT}'

TASKS = ['interactable', 'semantics', 'interaction']
IS_FINETUNE = ['', '_finetune']
FORMATS = ['det', 'seg']
METHODS = ['CenterNet2', 'FasterRCNN', 'MaskRCNN', 'YOLO', 'UIED', 'Xianyu',
           'GPT4V-E2E', 'Gemini-E2E', 'OmniParser', 'Claude4_5-sonnet-E2E', 'Gemini-2_5-pro-E2E', 'GPT5-E2E', 'O4-E2E','Qwen3-VL-plus-E2E', 'internVL-E2E', 'Seed-E2E', 'CogVLM']
LLM_METHODS = ['GPT4V-E2E', 'Gemini-E2E', 'OmniParser', 'Claude4_5-sonnet-E2E', 'Gemini-2_5-pro-E2E', 'GPT5-E2E', 'O4-E2E', 'Qwen3-VL-plus-E2E', 'internVL-E2E', 'Seed-E2E', 'CogVLM']

EVAL_DIMENTION = ['i', 's']
GT_JSON = f'{SPLIT}.json'

evaluator_script = './evaluate_coco.py'
interactable_mask_evaluator_script = './evaluate_interactable_mask.py'


def main(args):
    for format in FORMATS:
        for task in TASKS:
            gt_path = pjoin(GTS_ROOT, format, task, GT_JSON)
            for method in METHODS:
                for is_finetune in IS_FINETUNE:
                    result_path = pjoin(RESULTS_ROOT, format, task, method + is_finetune + '.json')
                    if not os.path.exists(result_path):
                        continue
                    for eval_dimension in EVAL_DIMENTION:
                        if task == 'interactable' and not eval_dimension == 'i':
                            continue
                        if task == 'semantics' and method in LLM_METHODS and eval_dimension == 'i':
                            continue
                        eval_result_dir = pjoin(EVAL_ROOT, format, task)
                        os.makedirs(eval_result_dir, exist_ok=True)
                        eval_result_path = pjoin(eval_result_dir, method + is_finetune + f'_{eval_dimension}' + '.csv')
                        eval_summary_path = pjoin(eval_result_dir, method + is_finetune + f'_{eval_dimension}' + '_summary.txt')

                        if os.path.exists(eval_result_path):
                            print(f'{eval_result_path} exists, skipping...')
                            continue

                        with open(result_path, 'r') as f:
                            result = json.load(f)
                        cli = f'python {evaluator_script} ' + \
                            f'-d {eval_dimension} ' + \
                            f'-gt {gt_path} ' + \
                            f'-dt {result_path} ' + \
                            f'-i {"bbox" if format == "det" else "segm"} ' + \
                            f'-l {eval_result_path} ' + \
                            '-s ' + \
                            ('-n ' if not (method in LLM_METHODS and task == 'semantics') else '') + \
                            f'> {eval_summary_path}'
                        print(cli)
                        os.system(cli)
                    if format == 'seg' and task == 'interactable':
                        eval_result_path = pjoin(eval_result_dir, method + is_finetune + f'_mask' + '.txt')
                        cli = f'python {interactable_mask_evaluator_script} ' + \
                            f'-gt {gt_path} ' + \
                            f'-dt {result_path} ' + \
                            f'-l {eval_result_path} ' + \
                            '-n '
                        print(cli)
                        os.system(cli)


if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    args = parser.parse_args()
    main(args)