File size: 2,411 Bytes
6d35aff | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 | # Description: Evaluate baselines on the context dataset
#
# Usage: python context_eval_baselines.py
import argparse
import os
import subprocess
from os.path import join as pjoin
RESULTS_ROOT = './results/baselines/cat'
EVAL_ROOT = './eval_context/results'
GT_PATH = './gts/context/context/cat.json'
EVAL_DIMENSION = 's'
METHODS = list(dict.fromkeys([
'CenterNet2', 'FasterRCNN', 'YOLO', 'UIED', 'Xianyu',
# 'GPT4V-E2E', 'Gemini-E2E',
'OmniParser', 'CogVLM', 'Seed-E2E', 'Claude4_5-sonnet-E2E',
'Gemini-2_5-pro-E2E', 'GPT5-E2E', 'O4-E2E',
'Qwen3-VL-plus-E2E', 'internVL-E2E']))
LLM_METHODS = list(dict.fromkeys([
# 'GPT4V-E2E', 'Gemini-E2E',
'OmniParser', 'CogVLM', 'Seed-E2E', 'Claude4_5-sonnet-E2E',
'Gemini-2_5-pro-E2E', 'GPT5-E2E', 'O4-E2E',
'Qwen3-VL-plus-E2E', 'internVL-E2E']))
evaluator_script = './context_eval.py'
def validate_unique_methods(methods):
seen = set()
duplicates = []
for method in methods:
if method in seen:
duplicates.append(method)
seen.add(method)
if duplicates:
raise ValueError(f"Duplicate methods in context baseline config: {duplicates}")
def main(args):
validate_unique_methods(METHODS)
validate_unique_methods(LLM_METHODS)
format = 'det'
task = 'semantics'
for iou in [0.75, 0.8, 0.85, 0.9, 0.95]:
for method in METHODS:
result_path = pjoin(RESULTS_ROOT, format, task, method + '.json')
if not os.path.exists(result_path):
continue
eval_result_path = pjoin(EVAL_ROOT, method + f'@{iou:.2f}.csv')
if os.path.exists(eval_result_path):
print(f'{eval_result_path} exists, skipping...')
continue
cli = [
'python',
evaluator_script,
'-d',
EVAL_DIMENSION,
'-g',
GT_PATH,
'-p',
result_path,
'-o',
eval_result_path,
'-i',
str(iou),
]
if method not in LLM_METHODS:
cli.append('-n')
print(' '.join(cli))
subprocess.run(cli, check=True)
def build_parser():
return argparse.ArgumentParser()
if __name__ == '__main__':
parser = build_parser()
args = parser.parse_args()
main(args)
|