| import argparse |
| import asyncio |
| import copy as cp |
| import datetime |
| import json |
| import os |
| import shutil |
| import subprocess |
| import sys |
| from functools import partial |
| from numbers import Real |
| from pathlib import Path |
| from typing import List |
|
|
| import pandas as pd |
| from tabulate import tabulate |
|
|
|
|
| |
| def get_gpu_list(): |
| CUDA_VISIBLE_DEVICES = os.environ.get('CUDA_VISIBLE_DEVICES', '') |
| if CUDA_VISIBLE_DEVICES != '': |
| gpu_list = [int(x) for x in CUDA_VISIBLE_DEVICES.split(',')] |
| return gpu_list |
| try: |
| ps = subprocess.Popen(('nvidia-smi', '--list-gpus'), stdout=subprocess.PIPE) |
| output = subprocess.check_output(('wc', '-l'), stdin=ps.stdout) |
| return list(range(int(output))) |
| except Exception: |
| return [] |
|
|
|
|
| RANK = int(os.environ.get('RANK', 0)) |
| WORLD_SIZE = int(os.environ.get('WORLD_SIZE', 1)) |
| LOCAL_WORLD_SIZE = int(os.environ.get("LOCAL_WORLD_SIZE", 1)) |
| LOCAL_RANK = int(os.environ.get("LOCAL_RANK", 1)) |
|
|
| GPU_LIST = get_gpu_list() |
| if LOCAL_WORLD_SIZE > 1 and len(GPU_LIST): |
| NGPU = len(GPU_LIST) |
| assert NGPU >= LOCAL_WORLD_SIZE, "The number of processes should be less than or equal to the number of GPUs" |
| GPU_PER_PROC = NGPU // LOCAL_WORLD_SIZE |
| DEVICE_START_IDX = GPU_PER_PROC * LOCAL_RANK |
| CUDA_VISIBLE_DEVICES = [str(i) for i in GPU_LIST[DEVICE_START_IDX: DEVICE_START_IDX + GPU_PER_PROC]] |
| CUDA_VISIBLE_DEVICES = ','.join(CUDA_VISIBLE_DEVICES) |
| |
| os.environ['CUDA_VISIBLE_DEVICES'] = CUDA_VISIBLE_DEVICES |
| print( |
| f'RANK: {RANK}, LOCAL_RANK: {LOCAL_RANK}, WORLD_SIZE: {WORLD_SIZE},' |
| f'LOCAL_WORLD_SIZE: {LOCAL_WORLD_SIZE}, CUDA_VISIBLE_DEVICES: {CUDA_VISIBLE_DEVICES}' |
| ) |
|
|
|
|
| from vlmeval.api import LMDeployAPI |
| from vlmeval.config import supported_VLM |
| from vlmeval.dataset import build_dataset |
| from vlmeval.dataset.video_dataset_config import supported_video_datasets |
| from vlmeval.inference import infer_data_job |
| from vlmeval.inference_mt import infer_data_job_mt |
| from vlmeval.inference_video import infer_data_job_video |
| from vlmeval.smp import (MMBenchOfficialServer, build_eval_id, collect_run_benchmark_report, |
| get_eval_file_format, get_logger, get_pred_file_format, |
| get_pred_file_path, githash, is_prediction_complete, listinstr, load, |
| load_env, prepare_reuse_files, proxy_set, setup_logger, timestr, |
| upsert_dataset_status, upsert_run_status) |
| from vlmeval.utils.result_transfer import MMMU_result_transfer, MMTBench_result_transfer |
|
|
| logger = get_logger(__name__) |
|
|
|
|
| def _format_fail_rate(failed, total): |
| if failed is None or total is None or total <= 0: |
| return '-' |
| return f'{failed / total * 100:.2f}% ({failed}/{total})' |
|
|
|
|
| def _format_sigfig(value): |
| if value is None or isinstance(value, bool): |
| return '-' |
| if isinstance(value, Real): |
| try: |
| if pd.isna(value): |
| return '-' |
| except Exception: |
| pass |
| return f'{float(value):.4g}' |
| return str(value) |
|
|
|
|
| def _format_metric_field(value): |
| if value is None: |
| return '-' |
| if isinstance(value, Real) and not isinstance(value, bool): |
| return _format_sigfig(value) |
| if isinstance(value, (list, tuple, dict)): |
| return json.dumps(value, ensure_ascii=False) |
| return str(value) |
|
|
|
|
| def _iter_primary_metric_rows(row): |
| primary_metric = row.get('primary_metric') |
| primary_metric_value = row.get('primary_metric_value') |
|
|
| if isinstance(primary_metric, (list, tuple)): |
| if not primary_metric: |
| return [(None, None)] |
| value_map = primary_metric_value if isinstance(primary_metric_value, dict) else {} |
| return [(metric_name, value_map.get(metric_name)) for metric_name in primary_metric] |
|
|
| return [(primary_metric, primary_metric_value)] |
|
|
|
|
| def log_run_benchmark_report(run_dir): |
| rows = collect_run_benchmark_report(run_dir) |
| if not rows: |
| logger.info(f'No benchmark summary rows found in {Path(run_dir) / "status.json"}') |
| return |
|
|
| report_rows = [] |
| for row in rows: |
| metric_rows = _iter_primary_metric_rows(row) |
| eval_error = row['eval_error'] or '-' |
| if eval_error != '-' and len(str(eval_error)) > 120: |
| eval_error = f'{str(eval_error)[:120]}...' |
|
|
| for idx, (primary_metric, primary_metric_value) in enumerate(metric_rows): |
| report_rows.append({ |
| 'benchmark': row['benchmark'] if idx == 0 else '', |
| 'infer_fail_rate': _format_fail_rate(row['infer_failed'], row['infer_total']) if idx == 0 else '', |
| 'judge_fail_rate': _format_fail_rate(row['judge_failed'], row['judge_total']) if idx == 0 else '', |
| 'primary_metric': _format_metric_field(primary_metric), |
| 'primary_metric_value': _format_metric_field(primary_metric_value), |
| 'skip_reason': (row['skip_reason'] or '-') if idx == 0 else '', |
| 'eval_error': eval_error if idx == 0 else '', |
| }) |
|
|
| logger.info('Run Summary Report:') |
| logger.info('\n' + tabulate(report_rows, headers='keys')) |
|
|
|
|
| |
| def build_model_from_config(cfg, model_name, use_vllm=False): |
| import vlmeval.api |
| import vlmeval.vlm |
| ws_bak = os.environ.pop('WORLD_SIZE', None) |
|
|
| config = cp.deepcopy(cfg[model_name]) |
| if use_vllm: |
| config['use_vllm'] = use_vllm |
| if 'class' not in config: |
| return supported_VLM[model_name](**config) |
| cls_name = config.pop('class') |
| if hasattr(vlmeval.api, cls_name): |
| model = getattr(vlmeval.api, cls_name)(**config) |
| elif hasattr(vlmeval.vlm, cls_name): |
| model = getattr(vlmeval.vlm, cls_name)(**config) |
| else: |
| raise ValueError(f'Class {cls_name} is not supported in `vlmeval.api` or `vlmeval.vlm`') |
|
|
| if ws_bak: |
| os.environ['WORLD_SIZE'] = ws_bak |
| return model |
|
|
|
|
| def build_dataset_from_config(cfg, dataset_name): |
| import inspect |
|
|
| import vlmeval.dataset |
| config = cp.deepcopy(cfg[dataset_name]) |
| if config == {}: |
| return supported_video_datasets[dataset_name]() |
| assert 'class' in config |
| cls_name = config.pop('class') |
| if hasattr(vlmeval.dataset, cls_name): |
| cls = getattr(vlmeval.dataset, cls_name) |
| sig = inspect.signature(cls.__init__) |
| valid_params = {k: v for k, v in config.items() if k in sig.parameters} |
| if cls.MODALITY == 'VIDEO': |
| if valid_params.get('fps', 0) > 0 and valid_params.get('nframe', 0) > 0: |
| raise ValueError('fps and nframe should not be set at the same time') |
| if valid_params.get('fps', 0) <= 0 and valid_params.get('nframe', 0) <= 0: |
| raise ValueError('fps and nframe should be set at least one valid value') |
| return cls(**valid_params) |
| else: |
| raise ValueError(f'Class {cls_name} is not supported in `vlmeval.dataset`') |
|
|
|
|
| def build_model_from_base_url(args): |
| """Build LMDeployAPI model kwargs from command-line arguments. |
| |
| Used by both local and API modes when --base-url is specified. |
| Returns a dict suitable for LMDeployAPI(**kwargs) or partial(LMDeployAPI, **kwargs). |
| """ |
| model_args = dict( |
| model=args.model[0] if isinstance(args.model, list) else args.model, |
| api_base=f"{args.base_url.rstrip('/')}/chat/completions", |
| key=args.key, |
| custom_prompt=args.custom_prompt, |
| max_tokens=args.max_tokens, |
| retry=args.retry, |
| timeout=args.timeout, |
| temperature=args.temperature, |
| top_k=args.top_k, |
| top_p=args.top_p, |
| repetition_penalty=args.repetition_penalty, |
| verbose=args.verbose, |
| video_llm=args.video_llm, |
| local_media=args.local_media, |
| ) |
| model_args = {k: v for k, v in model_args.items() if v is not None} |
| if args.thinker: |
| logger.warning('[Deprecated] Use `--max-tokens` and `--timeout` directly.') |
| model_args.update(dict(timeout=args.timeout * 2, max_tokens=args.max_tokens * 2)) |
| if args.extra_body: |
| try: |
| extra = json.loads(args.extra_body) |
| except Exception as e: |
| raise ValueError(f'Unable to parse the --extra-body value `{args.extra_body}`') from e |
| assert isinstance(extra, dict), '--extra-body must be a valid Python dict' |
| model_args.update(extra) |
| return model_args |
|
|
|
|
| def get_judge_kwargs(dataset_name, dataset_type, args): |
| """Determine judge kwargs based on dataset name and type. |
| |
| Uses run.py's logic as the canonical source for dataset-specific judge model |
| assignments, with additional entries from run_api.py (Video-MME). |
| Supports both local and API modes with mode-specific fallbacks. |
| """ |
| |
| if args.judge_api_nproc is not None: |
| nproc = args.judge_api_nproc |
| else: |
| nproc = args.api_nproc |
|
|
| |
| if args.judge_retry is not None: |
| retry = args.judge_retry |
| else: |
| retry = args.retry |
|
|
| judge_kwargs = { |
| 'nproc': nproc, |
| 'verbose': args.verbose, |
| 'retry': retry, |
| 'timeout': args.judge_timeout, |
| **(json.loads(args.judge_args) if args.judge_args else {}), |
| } |
|
|
| if args.judge_base_url: |
| judge_kwargs['api_base'] = f"{args.judge_base_url.rstrip('/')}/chat/completions" |
| if args.judge_key: |
| judge_kwargs['key'] = args.judge_key |
|
|
| if args.judge is not None: |
| judge_kwargs['model'] = args.judge |
| else: |
| if dataset_type in ['MCQ', 'Y/N', 'MCQ_MMMU_Pro'] or listinstr( |
| ['moviechat1k', 'mme-reasoning'], dataset_name.lower() |
| ): |
| if listinstr(['WeMath', 'MME-Reasoning'], dataset_name): |
| judge_kwargs['model'] = 'gpt-4o-mini' |
| elif listinstr(['VisualPuzzles'], dataset_name): |
| judge_kwargs['model'] = 'exact_matching' |
| elif listinstr(['PuzzleVQA'], dataset_name): |
| judge_kwargs['model'] = 'exact_matching' |
| elif listinstr(['VisuLogic'], dataset_name): |
| judge_kwargs['model'] = 'exact_matching' |
| else: |
| judge_kwargs['model'] = 'gpt-4o-mini' |
| elif listinstr(['MMVet', 'LLaVABench', 'MMBench_Video'], dataset_name): |
| if listinstr(['LLaVABench_KO'], dataset_name): |
| judge_kwargs['model'] = 'gpt-4o-0806' |
| else: |
| judge_kwargs['model'] = 'gpt-4-turbo' |
| elif listinstr(['VGRPBench'], dataset_name): |
| judge_kwargs['model'] = 'gpt-4o' |
| elif listinstr( |
| ['MathVista', 'MathVerse', 'MathVision', 'LENS', 'DynaMath', 'VL-RewardBench', |
| 'LogicVista', 'MOAT', 'OCR_Reasoning', 'VTCBench', 'Asclepius', |
| 'MMSafetyBench', 'MSSBench', 'SIUO', 'SIUO_GEN', 'XSTest', 'Flames'], dataset_name |
| ): |
| judge_kwargs['model'] = 'gpt-4o-mini' |
| elif listinstr(['OlympiadBench'], dataset_name): |
| use_api_judger = judge_kwargs.get("olympiad_use_api_judger", False) |
| if use_api_judger: |
| judge_kwargs['model'] = 'gpt-4o-mini' |
| elif listinstr( |
| ['MMLongBench', 'MMDU', 'DUDE', 'SLIDEVQA', 'MIA-Bench', |
| 'WildVision', 'MMAlignBench', 'MM-IFEval'], dataset_name |
| ): |
| judge_kwargs['model'] = 'gpt-4o' |
| elif listinstr(['ChartMimic'], dataset_name): |
| judge_kwargs['model'] = 'gpt-4o' |
| elif listinstr(['VDC'], dataset_name): |
| judge_kwargs['model'] = 'llama31-8b' |
| elif listinstr(['Video_MMLU_QA', 'Video_MMLU_CAP'], dataset_name): |
| judge_kwargs['model'] = 'qwen-72b' |
| elif listinstr(['MMVMBench'], dataset_name): |
| judge_kwargs['model'] = 'gpt-4o' |
| elif listinstr(['CVQA_EN', 'CVQA_LOC'], dataset_name): |
| judge_kwargs['model'] = 'gpt-4.1' |
| elif listinstr(['M4Bench'], dataset_name): |
| judge_kwargs['model'] = 'gpt-4o' |
| elif listinstr(['AyaVisionBench'], dataset_name): |
| judge_kwargs['model'] = 'gpt-4.1' |
| elif listinstr(['MathCanvas'], dataset_name): |
| judge_kwargs['model'] = 'gpt-4.1-2025-04-14' |
| elif listinstr(['MMReason'], dataset_name): |
| judge_kwargs['model'] = 'gpt-4.1' |
| elif listinstr(['CoreCognition'], dataset_name): |
| judge_kwargs['model'] = 'gpt-4.1' |
| elif listinstr(['WorldVQA'], dataset_name): |
| judge_kwargs['model'] = 'gpt-4o-1120' |
| elif listinstr(['Video-MME'], dataset_name): |
| judge_kwargs['model'] = 'gpt-4o-mini' |
| elif listinstr(['MaCBench'], dataset_name): |
| judge_kwargs['model'] = 'gpt-4o-mini' |
| elif listinstr(['SciDocBench'], dataset_name): |
| judge_kwargs['model'] = 'gpt-4o-mini' |
|
|
| if args.use_verifier: |
| judge_kwargs['use_verifier'] = True |
| if args.use_vllm: |
| judge_kwargs['use_vllm'] = True |
|
|
| return judge_kwargs |
|
|
|
|
| def parse_reuse_aux_arg(reuse_aux): |
| if isinstance(reuse_aux, bool): |
| return 'all' if reuse_aux else 'none' |
| if isinstance(reuse_aux, int): |
| return 'all' if reuse_aux else 'none' |
| if isinstance(reuse_aux, str): |
| value = reuse_aux.strip().lower() |
| if value in ['all', 'infer', 'none']: |
| return value |
| if value in ['1', 'true', 'yes']: |
| return 'all' |
| if value in ['0', 'false', 'no']: |
| return 'none' |
| raise argparse.ArgumentTypeError('reuse_aux must be one of: all, infer, none') |
|
|
|
|
| def parse_args(): |
| help_msg = """\ |
| You can launch the evaluation by setting either --data and --model or --config. |
| |
| --data and --model: |
| Each Arg should be a list of strings, specifying the names of datasets and models. |
| To find all supported model names, please refer to the `vlmeval/config.py` of check the output of the command \ |
| `vlmutil mlist all` in the terminal (you should first have vlmeval installed). |
| To find all supported dataset names, please refer to the `vlmeval/dataset/__init__.py` file. The python script \ |
| to print all supported dataset names is as follows: |
| ```python |
| from vlmeval.dataset import SUPPORTED_DATASETS |
| print(SUPPORTED_DATASETS) |
| ``` |
| or you can check the output of the command `vlmutil dlist all` in the terminal. |
| To find all supported video dataset default settings, please refer to the \ |
| `vlmeval/dataset/video_dataset_config.py` file. |
| |
| --config: |
| Launch the evaluation by specifying the path to the config json file. Sample Json Content: |
| ```json |
| { |
| "model": { |
| "GPT4o_20240806_T00_HIGH": { |
| "class": "GPT4V", |
| "model": "gpt-4o-2024-08-06", |
| "temperature": 0, |
| "img_detail": "high" |
| }, |
| "GPT4o_20240806_T10_Low": { |
| "class": "GPT4V", |
| "model": "gpt-4o-2024-08-06", |
| "temperature": 1.0, |
| "img_detail": "low" |
| }, |
| "GPT4o_20241120": {} |
| }, |
| "data": { |
| "MME-RealWorld-Lite": { |
| "class": "MMERealWorld", |
| "dataset": "MME-RealWorld-Lite" |
| }, |
| "MMBench_DEV_EN_V11": { |
| "class": "ImageMCQDataset", |
| "dataset": "MMBench_DEV_EN_V11" |
| }, |
| "MMBench_Video_8frame_nopack": {}, |
| "Video-MME_16frame_subs": { |
| "class": "VideoMME", |
| "dataset": "Video-MME", |
| "nframe": 16, |
| "use_subtitle": true, |
| } |
| } |
| } |
| ``` |
| Currently, only `model` and `data` are supported fields. The content of each field is a dictionary. |
| For `model`, the key is the name of the model, and the value is a dictionary containing the following keys: |
| - `class`: The class name of the model, which should be a class in `vlmeval.vlm` or `vlmeval.api`. |
| - Other keys are specific to the model, please refer to the corresponding class. |
| - Tip: The defined model in the `supported_VLM` of `vlmeval/config.py` can be used as a shortcut. |
| For `data`, the key is the name of the dataset (should be the same as the `dataset` field in most cases, \ |
| except for video datasets), and the value is a dictionary containing the following keys: |
| - `class`: The class name of the dataset, which should be a class in `vlmeval.dataset`. |
| - `dataset`: The name of the dataset, which should be a string that is accepted by the `dataset` argument of the \ |
| corresponding class. |
| - Other keys are specific to the dataset, please refer to the corresponding class. |
| - Tip: The defined dataset in the `supported_video_datasets` of `vlmeval/dataset/video_dataset_config.py` \ |
| can be used as a shortcut. |
| |
| The keys in the `model` and `data` fields will be used for naming the prediction files and evaluation results. |
| When launching with `--config`, args for API VLMs, such as `--retry`, `--verbose`, will be ignored. |
| |
| --api-mode: |
| Switch to the async API pipeline mode (originally run_api.py). This mode uses an optimized pipeline |
| for API-based models with cross-dataset unified inference queue, parallel inference and evaluation, |
| and better remote model utilization. |
| """ |
| parser = argparse.ArgumentParser(description=help_msg, formatter_class=argparse.RawTextHelpFormatter) |
|
|
| |
| parser.add_argument('--data', type=str, nargs='+', help='Names of Datasets') |
| parser.add_argument('--model', type=str, nargs='+', help='Names of Models') |
| parser.add_argument('--config', type=str, help='Path to the Config Json File') |
|
|
| |
| parser.add_argument('--work-dir', type=str, default='./outputs', help='select the output directory') |
| parser.add_argument('--mode', type=str, default='all', choices=['all', 'infer', 'eval']) |
|
|
| |
| parser.add_argument('--api-nproc', type=int, default=32, help='Parallel API calling') |
| parser.add_argument('--retry', type=int, default=6, help='retry numbers for API VLMs') |
| parser.add_argument('--verbose', action='store_true') |
| parser.add_argument('--keep-failed', action='store_true', |
| help='Keep failed predictions as-is instead of retrying them.') |
| parser.add_argument( |
| '--ignore', |
| action='store_true', |
| help='[Deprecated] Ignore failed indices, it is the default behavior now. ' |
| 'Use `--keep-failed` to disable it.') |
| parser.add_argument('--reuse', action='store_true') |
| parser.add_argument( |
| '--reuse-aux', |
| type=parse_reuse_aux_arg, |
| default='all', |
| help='Reuse auxiliary files: `all` for infer+eval aux, `infer` for inference-only aux, `none` for no aux.' |
| ) |
| parser.add_argument( |
| '--use-vllm', action='store_true', help='use vllm to generate, the flag is only supported in Llama4 for now') |
| parser.add_argument('--use-verifier', action='store_true', help='use verifier to evaluate') |
|
|
| |
| parser.add_argument('--judge', type=str, default=None) |
| parser.add_argument('--judge-args', type=str, default=None, help='Judge arguments in JSON format') |
| parser.add_argument('--judge-base-url', type=str, default=None, help='Base URL of judge API') |
| parser.add_argument('--judge-key', type=str, default=None, help='API key for judge model') |
| parser.add_argument('--judge-api-nproc', type=int, default=None, |
| help='Parallel API calling for judger (defaults to follow --api-nproc)') |
| parser.add_argument('--judge-retry', type=int, default=None, |
| help='Retry times for failed judgement (defaults to follow --retry)') |
| parser.add_argument('--judge-timeout', type=int, default=600, |
| help='Max time in seconds for judgement.') |
|
|
| |
| parser.add_argument('--base-url', type=str, default=None, |
| help='Base URL of OpenAI-compatible API (e.g. http://localhost:8080/v1). ' |
| 'If set, LMDeployAPI is used for inference without modifying config.py.') |
| parser.add_argument('--key', type=str, default='sk-admin', help='API key for inference model') |
| parser.add_argument('--thinker', action='store_true', |
| help='[Deprecated] Enable thinking mode: doubles timeout and max_tokens.') |
| parser.add_argument('--max-tokens', type=int, default=2 ** 15, |
| help='Max tokens for model generation.') |
| parser.add_argument('--temperature', type=float, default=None) |
| parser.add_argument('--top-k', type=int, default=None) |
| parser.add_argument('--top-p', type=float, default=None) |
| parser.add_argument('--repetition-penalty', type=float, default=None) |
| parser.add_argument('--timeout', type=int, default=1800, |
| help='Max time in seconds for a single inference request.') |
| parser.add_argument('--custom-prompt', type=str, default=None, |
| help='Manually select a model adapter by name.') |
| parser.add_argument('--extra-body', type=str, default=None, |
| help='Extra inference parameters as json dict string') |
| parser.add_argument('--video-llm', action='store_true', |
| help='Whether the API model supports native video inputs.') |
| parser.add_argument('--local-media', action='store_true', |
| help='Whether to send local media file path to the API model.') |
|
|
| |
| parser.add_argument('--api-mode', action='store_true', |
| help='Switch to async API pipeline mode') |
| parser.add_argument('--monitor-interval', type=int, default=30, |
| help='Status monitoring interval in seconds') |
| parser.add_argument('--debug', action='store_true', |
| help='Debug mode: run evaluation in main process') |
|
|
| args = parser.parse_args() |
| if args.ignore: |
| logger.warning('[Deprecated] the `--ignore` flag is deprecated since it is ' |
| 'the default behavior, use `--keep-failed` to disable it.') |
| return args |
|
|
|
|
| def run_local_mode(args): |
| """Original evaluation mode with GPU/distributed support.""" |
| use_config, cfg = False, None |
| if args.config is not None: |
| assert args.data is None and args.model is None, '--data and --model should not be set when using --config' |
| use_config, cfg = True, load(args.config) |
| args.model = list(cfg['model'].keys()) |
| args.data = list(cfg['data'].keys()) |
| else: |
| assert len(args.data), '--data should be a list of data files' |
|
|
| if 'MMEVAL_ROOT' in os.environ: |
| args.work_dir = os.environ['MMEVAL_ROOT'] |
|
|
| commit_id = githash(digits=8) |
| eval_id = build_eval_id() |
| setup_logger(log_file=os.path.join(args.work_dir, 'logs', f'{eval_id}_{timestr()}.log')) |
|
|
| if args.mode == 'eval': |
| args.reuse = True |
| logger.info('Force to use `reuse=True` for eval mode.') |
|
|
| if RANK == 0: |
| if not args.reuse: |
| logger.warning('--reuse is not set, this run will start from a fresh output directory') |
| else: |
| logger.info(f'--reuse is set, reuse-aux={args.reuse_aux}') |
|
|
| if not use_config: |
| for k, v in supported_VLM.items(): |
| if hasattr(v, 'keywords') and 'retry' in v.keywords and args.retry is not None: |
| v.keywords['retry'] = args.retry |
| supported_VLM[k] = v |
| if hasattr(v, 'keywords') and 'verbose' in v.keywords and args.verbose is not None: |
| v.keywords['verbose'] = args.verbose |
| supported_VLM[k] = v |
|
|
| |
| if os.environ.get('FWD_API', None) == '1': |
| from vlmeval.api import GPT4V |
| from vlmeval.config import api_models as supported_APIs |
| for m in args.model: |
| if m in supported_APIs: |
| kws = supported_VLM[m].keywords |
| supported_VLM[m] = partial(GPT4V, **kws) |
| logger.warning(f'FWD_API is set, will use class `GPT4V` for {m}') |
|
|
| if WORLD_SIZE > 1: |
| import torch.distributed as dist |
| dist.init_process_group( |
| backend='nccl', |
| timeout=datetime.timedelta(seconds=int(os.environ.get('DIST_TIMEOUT', 3600))) |
| ) |
|
|
| for _, model_name in enumerate(args.model): |
| logger.info(f'=========== {model_name} ===========') |
| model = None |
|
|
| pred_root_meta = Path(args.work_dir) / model_name |
| pred_root = pred_root_meta / eval_id |
| pred_root_meta.mkdir(parents=True, exist_ok=True) |
| pred_root.mkdir(parents=True, exist_ok=True) |
|
|
| if RANK == 0: |
| upsert_run_status( |
| pred_root, |
| eval_id=eval_id, |
| created_at=datetime.datetime.now().astimezone().isoformat(), |
| commit=commit_id, |
| argv=sys.argv, |
| api_mode=False, |
| world_size=WORLD_SIZE, |
| pred_format=get_pred_file_format(), |
| eval_format=get_eval_file_format(), |
| mode=args.mode, |
| reuse=bool(args.reuse), |
| reuse_aux=args.reuse_aux, |
| model_name=model_name, |
| ) |
|
|
| if use_config: |
| model = build_model_from_config(cfg['model'], model_name, args.use_vllm) |
| elif args.base_url: |
| model_args = build_model_from_base_url(args) |
| model_args['model'] = model_name |
| model = LMDeployAPI(**model_args) |
|
|
| for _, dataset_name in enumerate(args.data): |
| logger.info(f'----------- {dataset_name} -----------') |
| if WORLD_SIZE > 1: |
| dist.barrier() |
|
|
| dataset = None |
| result_file = None |
| judge_model = None |
|
|
| try: |
| result_file = get_pred_file_path( |
| str(pred_root), model_name, dataset_name, use_env_format=True) |
| if RANK == 0: |
| upsert_dataset_status( |
| run_dir=pred_root, |
| model_name=model_name, |
| dataset_name=dataset_name, |
| prediction_file=result_file, |
| status='pending', |
| ) |
|
|
| if use_config: |
| if WORLD_SIZE > 1: |
| if RANK == 0: |
| dataset = build_dataset_from_config(cfg['data'], dataset_name) |
| dist.barrier() |
| dataset = build_dataset_from_config(cfg['data'], dataset_name) |
| if dataset is None: |
| logger.error(f'Dataset {dataset_name} is not valid, will be skipped. ') |
| if RANK == 0: |
| upsert_dataset_status( |
| run_dir=pred_root, |
| model_name=model_name, |
| dataset_name=dataset_name, |
| status='done', |
| skip_reason='invalid_dataset', |
| ) |
| continue |
| else: |
| dataset_kwargs = {} |
| if dataset_name in ['MMLongBench_DOC', 'DUDE', 'DUDE_MINI', 'SLIDEVQA', 'SLIDEVQA_MINI']: |
| dataset_kwargs['model'] = model_name |
|
|
| |
| if WORLD_SIZE > 1: |
| if RANK == 0: |
| dataset = build_dataset(dataset_name, **dataset_kwargs) |
| dist.barrier() |
|
|
| dataset = build_dataset(dataset_name, **dataset_kwargs) |
| if dataset is None: |
| logger.error(f'Dataset {dataset_name} is not valid, will be skipped. ') |
| if RANK == 0: |
| upsert_dataset_status( |
| run_dir=pred_root, |
| model_name=model_name, |
| dataset_name=dataset_name, |
| status='done', |
| skip_reason='invalid_dataset', |
| ) |
| continue |
|
|
| judge_kwargs = get_judge_kwargs(dataset_name, dataset.TYPE, args) |
| judge_model = judge_kwargs.get('model', '') |
|
|
| if RANK == 0: |
| reuse_ctx = prepare_reuse_files( |
| pred_root_meta=pred_root_meta, |
| eval_id=eval_id, |
| model_name=model_name, |
| dataset_name=dataset_name, |
| dataset=dataset, |
| result_file=result_file, |
| reuse=args.reuse, |
| reuse_aux=args.reuse_aux, |
| retry_failed=not args.keep_failed, |
| judge_model=judge_model if args.mode != 'infer' else None, |
| world_size=WORLD_SIZE, |
| ) |
| upsert_dataset_status( |
| run_dir=pred_root, |
| model_name=model_name, |
| dataset_name=dataset_name, |
| source_run=reuse_ctx['source_eval_id'], |
| judge_model=judge_model, |
| reuse_aux=args.reuse_aux, |
| ) |
| logger.info(judge_kwargs) |
|
|
| if WORLD_SIZE > 1: |
| dist.barrier() |
|
|
| prediction_complete = is_prediction_complete( |
| result_file, |
| dataset_indices=list(dataset.data['index']), |
| retry_failed=not args.keep_failed, |
| ) |
| if args.mode == 'eval' and not prediction_complete: |
| if RANK == 0: |
| logger.error( |
| f'No reusable completed prediction found for {model_name} x {dataset_name}, ' |
| 'skipping this combination in eval mode.' |
| ) |
| if Path(result_file).exists(): |
| skip_reason = 'Incomplete infer result' |
| else: |
| skip_reason = 'No infer result found' |
| upsert_dataset_status( |
| run_dir=pred_root, |
| model_name=model_name, |
| dataset_name=dataset_name, |
| status='done', |
| skip_reason=skip_reason, |
| ) |
| continue |
|
|
| if model is None: |
| model = model_name |
|
|
| if args.mode != "eval": |
| if RANK == 0: |
| upsert_dataset_status( |
| run_dir=pred_root, |
| model_name=model_name, |
| dataset_name=dataset_name, |
| status='infer', |
| ) |
| |
| if dataset.MODALITY == 'VIDEO': |
| model = infer_data_job_video( |
| model, |
| work_dir=pred_root, |
| model_name=model_name, |
| dataset=dataset, |
| result_file=result_file, |
| verbose=args.verbose, |
| api_nproc=args.api_nproc, |
| use_vllm=args.use_vllm, |
| retry_failed=not args.keep_failed) |
| elif dataset.TYPE == 'MT': |
| model = infer_data_job_mt( |
| model, |
| work_dir=pred_root, |
| model_name=model_name, |
| dataset=dataset, |
| verbose=args.verbose, |
| api_nproc=args.api_nproc, |
| retry_failed=not args.keep_failed, |
| use_vllm=args.use_vllm) |
| else: |
| model = infer_data_job( |
| model, |
| work_dir=pred_root, |
| model_name=model_name, |
| dataset=dataset, |
| verbose=args.verbose, |
| api_nproc=args.api_nproc, |
| retry_failed=not args.keep_failed, |
| use_vllm=args.use_vllm) |
|
|
| if WORLD_SIZE > 1: |
| dist.barrier() |
|
|
| |
| if RANK == 0: |
| if args.mode != 'infer': |
| upsert_dataset_status( |
| run_dir=pred_root, |
| model_name=model_name, |
| dataset_name=dataset_name, |
| status='eval', |
| ) |
| |
| if dataset_name in ['MMMU_TEST']: |
| result_json = MMMU_result_transfer(result_file) |
| logger.info(f'Transfer MMMU_TEST result to json for official evaluation, ' |
| f'json file saved in {result_json}') |
| upsert_dataset_status( |
| run_dir=pred_root, |
| model_name=model_name, |
| dataset_name=dataset_name, |
| status='done', |
| skip_reason='official_submission_only_mmmu_test', |
| ) |
| continue |
| elif 'MMT-Bench_ALL' in dataset_name: |
| submission_file = MMTBench_result_transfer(result_file, **judge_kwargs) |
| logger.info(f'Extract options from prediction of MMT-Bench FULL split for official evaluation ' |
| f'(https://eval.ai/web/challenges/challenge-page/2328/overview), ' |
| f'submission file saved in {submission_file}') |
| upsert_dataset_status( |
| run_dir=pred_root, |
| model_name=model_name, |
| dataset_name=dataset_name, |
| status='done', |
| skip_reason='official_submission_only_mmt_bench', |
| ) |
| continue |
|
|
| |
| if args.mode == 'infer': |
| upsert_dataset_status( |
| run_dir=pred_root, |
| model_name=model_name, |
| dataset_name=dataset_name, |
| status='done', |
| skip_reason='mode_infer', |
| ) |
| continue |
|
|
| |
| if 'MLLMGuard_DS' in dataset_name: |
| logger.info('The evaluation of MLLMGuard_DS is not supported yet. ') |
| upsert_dataset_status( |
| run_dir=pred_root, |
| model_name=model_name, |
| dataset_name=dataset_name, |
| status='done', |
| skip_reason='evaluation_not_supported_for_dataset', |
| ) |
| continue |
| elif 'AesBench_TEST' == dataset_name: |
| logger.info(f'The results are saved in {result_file}. ' |
| f'Please send it to the AesBench Team via huangyipo@hotmail.com.') |
| upsert_dataset_status( |
| run_dir=pred_root, |
| model_name=model_name, |
| dataset_name=dataset_name, |
| status='done', |
| skip_reason='external_submission_required', |
| ) |
| continue |
| elif dataset_name in ['DocVQA_TEST', 'InfoVQA_TEST', 'Q-Bench1_TEST', 'A-Bench_TEST']: |
| logger.info(f'{dataset_name} is a test split without ground-truth. ' |
| 'Thus only the inference part is supported for those datasets. ') |
| upsert_dataset_status( |
| run_dir=pred_root, |
| model_name=model_name, |
| dataset_name=dataset_name, |
| status='done', |
| skip_reason='test_split_without_ground_truth', |
| ) |
| continue |
| elif dataset_name in [ |
| 'MMBench_TEST_CN', 'MMBench_TEST_EN', 'MMBench', 'MMBench_CN', |
| 'MMBench_TEST_CN_V11', 'MMBench_TEST_EN_V11', 'MMBench_V11', 'MMBench_CN_V11' |
| ] and not MMBenchOfficialServer(dataset_name): |
| logger.error( |
| f'Can not evaluate {dataset_name} on non-official servers, will skip the evaluation.') |
| upsert_dataset_status( |
| run_dir=pred_root, |
| model_name=model_name, |
| dataset_name=dataset_name, |
| status='done', |
| skip_reason='mmbench_evaluation_requires_official_server', |
| ) |
| continue |
|
|
| |
| eval_proxy = os.environ.get('EVAL_PROXY', None) |
| old_proxy = os.environ.get('HTTP_PROXY', '') |
| if eval_proxy is not None: |
| proxy_set(eval_proxy) |
|
|
| |
| eval_results = dataset.evaluate(result_file, **judge_kwargs) |
| |
| if eval_results is not None: |
| summary_eval_results = eval_results |
| assert isinstance(eval_results, dict) or isinstance(eval_results, pd.DataFrame) |
| logger.info(f'The evaluation of model {model_name} x dataset {dataset_name} has finished! ') |
| logger.info('Evaluation Results:') |
| if isinstance(eval_results, dict): |
| logger.info('\n' + json.dumps(eval_results, indent=4)) |
| elif isinstance(eval_results, pd.DataFrame): |
| if len(eval_results) < len(eval_results.columns): |
| eval_results = eval_results.T |
| logger.info('\n' + tabulate(eval_results)) |
| upsert_dataset_status( |
| run_dir=pred_root, |
| model_name=model_name, |
| dataset_name=dataset_name, |
| status='done', |
| metrics_source=summary_eval_results, |
| dataset_obj=dataset, |
| ) |
| else: |
| upsert_dataset_status( |
| run_dir=pred_root, |
| model_name=model_name, |
| dataset_name=dataset_name, |
| status='done', |
| skip_reason='evaluate_returned_none', |
| ) |
|
|
| |
| if eval_proxy is not None: |
| proxy_set(old_proxy) |
|
|
| |
| files = [ |
| path for path in pred_root.iterdir() |
| if path.is_file() and ( |
| f'{model_name}_{dataset_name}' in path.name or path.name == 'status.json' |
| ) |
| ] |
| |
| files = [ |
| path for path in files |
| if not path.name.endswith(('_checkpoint.pkl', '_PREV.pkl', '_structs.pkl')) |
| ] |
| for file_addr in files: |
| link_addr = pred_root_meta / file_addr.name |
| if link_addr.is_dir() and not link_addr.is_symlink(): |
| shutil.rmtree(link_addr) |
| elif link_addr.exists() or link_addr.is_symlink(): |
| link_addr.unlink() |
| rel_target = file_addr.relative_to(pred_root_meta) |
| link_addr.symlink_to(rel_target) |
|
|
| except Exception as e: |
| logger.exception(f'Model {model_name} x Dataset {dataset_name} combination failed: {e}, ' |
| 'skipping this combination.') |
| if RANK == 0: |
| upsert_dataset_status( |
| run_dir=pred_root, |
| model_name=model_name, |
| dataset_name=dataset_name, |
| status='done', |
| error_message=str(e), |
| ) |
| continue |
|
|
| if RANK == 0: |
| log_run_benchmark_report(pred_root) |
|
|
| if WORLD_SIZE > 1: |
| dist.destroy_process_group() |
|
|
|
|
| def run_api_mode(args): |
| """Async API pipeline mode for API-based models. |
| |
| Uses an optimized pipeline with cross-dataset unified inference queue, |
| parallel inference and evaluation, and better remote model utilization. |
| """ |
| from vlmeval.api.adapters import get_adapter_registry |
| from vlmeval.inference_api import APIEvalPipeline, DatasetConfig |
|
|
| |
| if isinstance(args.model, list): |
| if len(args.model) > 1: |
| raise ValueError('API mode only supports a single model. Got: ' + str(args.model)) |
| args.model = args.model[0] |
|
|
| |
| if args.custom_prompt: |
| registry = get_adapter_registry() |
| assert args.custom_prompt in registry, \ |
| f'Unknown adapter: {args.custom_prompt}. Available: {list(registry.keys())}' |
|
|
| assert args.data, '--data must be set in API mode' |
|
|
| |
| commit_id = githash(digits=8) |
| eval_id = build_eval_id() |
| model_name = args.model.replace('/', '--') |
|
|
| work_dir = Path(args.work_dir) / model_name |
| work_dir.mkdir(parents=True, exist_ok=True) |
|
|
| pred_root = Path(args.work_dir) / model_name / eval_id |
| pred_root.mkdir(exist_ok=True) |
|
|
| log_file = Path(work_dir) / 'logs' / f'{eval_id}_{datetime.datetime.now().strftime("%H%M%S")}.log' |
| setup_logger(log_file=str(log_file)) |
| logger.info(f'Log file: {log_file}') |
|
|
| if args.mode == 'eval': |
| args.reuse = True |
| logger.info('Force to use `reuse=True` for eval mode.') |
|
|
| if not args.reuse: |
| logger.warning('--reuse is not set, this run will start from a fresh output directory') |
| else: |
| logger.info(f'--reuse is set, reuse-aux={args.reuse_aux}') |
|
|
| WORLD_SIZE_LOCAL = int(os.environ.get('WORLD_SIZE', 1)) |
| if WORLD_SIZE_LOCAL > 1: |
| logger.error("API pipeline does not support multi-process mode (WORLD_SIZE > 1).") |
| return |
|
|
| |
| if args.base_url is not None: |
| model_args = build_model_from_base_url(args) |
| model_builder = partial(LMDeployAPI, **model_args) |
| else: |
| assert model_name in supported_VLM, \ |
| f'Model "{model_name}" not found in supported_VLM. Consider using --base-url to specify an API endpoint.' |
| model_builder = supported_VLM[model_name] |
|
|
| upsert_run_status( |
| pred_root, |
| eval_id=eval_id, |
| created_at=datetime.datetime.now().astimezone().isoformat(), |
| commit=commit_id, |
| argv=sys.argv, |
| api_mode=True, |
| world_size=1, |
| pred_format=get_pred_file_format(), |
| eval_format=get_eval_file_format(), |
| mode=args.mode, |
| reuse=bool(args.reuse), |
| reuse_aux=args.reuse_aux, |
| model_name=model_name, |
| ) |
|
|
| |
| dataset_configs: List[DatasetConfig] = [] |
|
|
| for ds_name in args.data: |
| logger.info(f'-------------------- {ds_name} --------------------') |
|
|
| try: |
| dataset_kwargs = {} |
| if ds_name in [ |
| 'MMLongBench_DOC', 'DUDE', 'DUDE_MINI', |
| 'SLIDEVQA', 'SLIDEVQA_MINI', |
| ]: |
| dataset_kwargs['model'] = model_name |
| dataset = build_dataset(ds_name, **dataset_kwargs) |
|
|
| if dataset is None: |
| logger.error(f'Dataset {ds_name} is not valid, will be skipped.') |
| continue |
|
|
| |
| result_file = get_pred_file_path( |
| pred_root, model_name, ds_name, use_env_format=True) |
|
|
| |
| if ds_name in ['MMMU_TEST']: |
| logger.info(f'{ds_name} requires special handling, skipped in pipeline.') |
| continue |
| if 'MMT-Bench_ALL' in ds_name: |
| logger.info(f'{ds_name} requires special handling, skipped in pipeline.') |
| continue |
|
|
| judge_kwargs = get_judge_kwargs(ds_name, dataset.TYPE, args) |
| judge_model = judge_kwargs.get('model', '') |
| logger.info(f'Judge kwargs: {judge_kwargs}') |
|
|
| reuse_ctx = prepare_reuse_files( |
| pred_root_meta=str(work_dir), |
| eval_id=eval_id, |
| model_name=model_name, |
| dataset_name=ds_name, |
| dataset=dataset, |
| result_file=result_file, |
| reuse=args.reuse, |
| reuse_aux=args.reuse_aux, |
| retry_failed=not args.keep_failed, |
| judge_model=judge_model if args.mode != 'infer' else None, |
| world_size=1, |
| ) |
| upsert_dataset_status( |
| run_dir=pred_root, |
| model_name=model_name, |
| dataset_name=ds_name, |
| prediction_file=result_file, |
| source_run=reuse_ctx['source_eval_id'], |
| judge_model=judge_model, |
| reuse_aux=args.reuse_aux, |
| ) |
| if args.mode == 'eval' and not reuse_ctx['prediction_complete']: |
| logger.error( |
| f'No reusable completed prediction found for {model_name} x {ds_name}, ' |
| 'skipping this dataset in eval mode.' |
| ) |
| try: |
| if Path(result_file).exists(): |
| skip_reason = 'Incomplete infer result' |
| else: |
| skip_reason = 'No infer result found' |
| upsert_dataset_status( |
| run_dir=pred_root, |
| model_name=model_name, |
| dataset_name=ds_name, |
| status='done', |
| skip_reason=skip_reason, |
| ) |
| except Exception as summary_err: |
| logger.warning( |
| f'Failed to update status.json for {model_name} x {ds_name}: {summary_err}' |
| ) |
| continue |
|
|
| |
| if dataset.MODALITY == 'VIDEO': |
| dataset_type = 'video' |
| elif dataset.TYPE == 'MT': |
| dataset_type = 'mt' |
| else: |
| dataset_type = 'image' |
| dataset_config = DatasetConfig( |
| dataset_name=ds_name, |
| dataset_obj=dataset, |
| dataset_type=dataset_type, |
| model_obj=model_builder(), |
| model_name=model_name, |
| work_dir=str(pred_root), |
| result_file=result_file, |
| judge_kwargs=judge_kwargs, |
| verbose=args.verbose |
| ) |
| dataset_configs.append(dataset_config) |
|
|
| except Exception as e: |
| logger.exception(f'Failed to prepare dataset {ds_name}: {e}') |
| continue |
|
|
| |
| if len(dataset_configs) == 0: |
| logger.warning('No valid datasets to evaluate.') |
| return |
|
|
| logger.info(f"Starting API Pipeline for model: {model_name}") |
| logger.info(f"Total datasets: {len(dataset_configs)}") |
|
|
| pipeline = APIEvalPipeline( |
| dataset_configs=dataset_configs, |
| concurrency=args.api_nproc, |
| monitor_interval=args.monitor_interval, |
| run_infer=args.mode in {'infer', 'all'}, |
| run_eval=args.mode in {'eval', 'all'}, |
| debug=args.debug, |
| retry_failed=not args.keep_failed |
| ) |
|
|
| try: |
| asyncio.run(pipeline.run()) |
| except KeyboardInterrupt: |
| logger.warning("Pipeline interrupted by user.") |
| except Exception as e: |
| logger.exception(f"Pipeline failed with error: {e}") |
| finally: |
| log_run_benchmark_report(pred_root) |
|
|
|
|
| def main(): |
| args = parse_args() |
| if args.api_mode: |
| run_api_mode(args) |
| else: |
| run_local_mode(args) |
|
|
|
|
| if __name__ == '__main__': |
| load_env() |
| main() |
|
|
